[2021-10-22T00:31:41.215+0000] [header.info] [::0] GitCommitId = 8ca6696edbaedcf764f86e89b5cf73097b6154f7 [2021-10-22T00:31:41.215+0000] [header.info] [::0] AutomationVersion = 11.7.0.7168 [2021-10-22T00:31:41.215+0000] [header.info] [::0] localhost = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com [2021-10-22T00:31:41.215+0000] [header.info] [::0] ErrorStateSleepTime = 10s [2021-10-22T00:31:41.215+0000] [header.info] [::0] GoalStateSleepTime = 10s [2021-10-22T00:31:41.215+0000] [header.info] [::0] NotGoalStateSleepTime = 1s [2021-10-22T00:31:41.215+0000] [header.info] [::0] PlanCutoffTime = 300000 [2021-10-22T00:31:41.215+0000] [header.info] [::0] TracePlanner = false [2021-10-22T00:31:41.215+0000] [header.info] [::0] User = mongod [2021-10-22T00:31:41.215+0000] [header.info] [::0] Go version = go1.16.7 [2021-10-22T00:31:41.216+0000] [.info] [main/components/agent.go:NewAgent:166] [00:31:41.216] Starting MMS Change Listener [2021-10-22T00:31:41.216+0000] [.info] [main/components/agent.go:NewAgent:182] [00:31:41.216] Started MMS Change Listener [2021-10-22T00:31:41.216+0000] [.info] [main/components/agent.go:NewAgent:184] [00:31:41.216] Starting MMS Job Manager [2021-10-22T00:31:41.235+0000] [.info] [main/components/agent.go:NewAgent:186] [00:31:41.235] Started MMS Job Manager [2021-10-22T00:31:41.235+0000] [.info] [src/mmssettings/mmssettings.go:Start:226] [00:31:41.235] Start MMS Settings gatherer [2021-10-22T00:31:41.235+0000] [.info] [src/mmsfeatures/mmsfeatures.go:newNotifier:43] [00:31:41.235] Started MMS cloud migration agent features notifier [2021-10-22T00:31:41.236+0000] [.info] [src/filebeat/filebeat.go:controlLoop:628] [00:31:41.236] Starting control loop [2021-10-22T00:31:41.237+0000] [.info] [src/dataexplorer/dataexplorer.go:controlLoop:108] [00:31:41.237] Starting control loop [2021-10-22T00:31:41.237+0000] [.info] [realtime/rtcollector/rtcollector.go:controlLoop:87] [00:31:41.237] Starting control loop [2021-10-22T00:31:41.238+0000] [.info] [src/kmipproxy/custodian.go:mainLoop:205] [00:31:41.238] Starting main loop [2021-10-22T00:31:41.399+0000] [.info] [src/filebeat/filebeat.go:stop:784] [00:31:41.399] Stopping filebeat log collection... [2021-10-22T00:31:41.399+0000] [.info] [src/filebeat/filebeat.go:stop:789] [00:31:41.399] Filebeat log collection stopped. [2021-10-22T00:31:41.795+0000] [.info] [src/config/config.go:ReadClusterConfig:509] [00:31:41.795] Backing up cluster config to /var/lib/mongodb-mms-automation/mms-cluster-config-backup.json [2021-10-22T00:31:41.795+0000] [.info] [main/components/agent.go:LoadClusterConfig:262] [00:31:41.795] New cluster config received! 0 () -> 42 ("692dea1cb47ae80ebf0db09c534b1de5--gzip") [2021-10-22T00:31:41.796+0000] [.info] [src/modules/agents_unix.go:KillAllAgents:42] [00:31:41.795] Killing all running mongodb-mms-monitoring-agent agents at /var/lib/mongodb-mms-automation/mongodb-mms-monitoring-agent-.+\..+_.+/mongodb-mms-monitoring-agent *$ [2021-10-22T00:31:41.835+0000] [.info] [src/modules/agents_unix.go:KillAllAgents:42] [00:31:41.835] Killing all running mongodb-mms-backup-agent agents at /var/lib/mongodb-mms-automation/mongodb-mms-backup-agent-.+\..+_.+/mongodb-mms-backup-agent *$ [2021-10-22T00:31:43.971+0000] [.info] [main/components/agent.go:launchNewDirectors:558] [00:31:43.971] Launched director atlas-4wnh1p-config-0-node-1 [2021-10-22T00:31:43.971+0000] [.info] [metrics/collector/realftscollector.go:mainLoop:239] [00:31:43.971] Received new cluster config! [2021-10-22T00:31:43.971+0000] [.info] [metrics/prometheus/exporter.go:mainLoop:73] [00:31:43.971] Received new cluster config! [2021-10-22T00:31:43.971+0000] [.info] [metrics/prometheus/server.go:ApplyNewPromConfig:53] [00:31:43.971] Prometheus config is nil or has prometheus explicitly disabled, stopping the prometheus server if running... [2021-10-22T00:31:43.971+0000] [.info] [metrics/prometheus/server.go:func1:51] [00:31:43.971] ApplyNewPromConfig took 22.975µs [2021-10-22T00:31:43.971+0000] [.info] [realtime/rtcollector/rtcollector.go:controlLoop:95] [00:31:43.971] Received new cluster config! [2021-10-22T00:31:43.971+0000] [.info] [metrics/collector/realcollector.go:collectLoop:253] [00:31:43.971] Received new cluster config! [2021-10-22T00:31:43.971+0000] [.info] [src/dataexplorer/dataexplorer.go:controlLoop:114] [00:31:43.971] Received new cluster config! [2021-10-22T00:31:43.971+0000] [.info] [src/filebeat/filebeat.go:controlLoop:634] [00:31:43.971] Received new cluster config! [2021-10-22T00:31:43.974+0000] [.info] [src/director/director.go:incorporateConfig:302] [00:31:43.974] clusterConfig edition is different ( -> "692dea1cb47ae80ebf0db09c534b1de5--gzip"). Incorporating... [2021-10-22T00:31:44.001+0000] [.info] [src/filebeat/filebeat.go:stop:784] [00:31:44.001] Stopping filebeat log collection... [2021-10-22T00:31:44.001+0000] [.info] [src/filebeat/filebeat.go:stop:789] [00:31:44.001] Filebeat log collection stopped. [2021-10-22T00:31:44.005+0000] [.info] [src/modules/maintainer.go:StartModule:295] [00:31:44.005] Starting Backup Module with 9 conf keys = [logFile maxLogFileSizeBytes sslTrustedServerCertificates mmsApiKey mmsGroupId mothership version https maxLogFileDurationHrs] [2021-10-22T00:31:44.011+0000] [.info] [src/modules/maintainer.go:StartModule:295] [00:31:44.011] Starting CPS Module with 11 conf keys = [confCallHost maxLogFileSizeBytes maxLogFileDurationHrs sslTrustedServerCertificates version mmsApiKey mmsGroupId confCallProtocolScheme maxRetries logFile clusterName] [2021-10-22T00:31:44.048+0000] [.info] [src/modules/maintainer.go:StartModule:315] [00:31:44.048] Started CPS Module [2021-10-22T00:31:44.064+0000] [.info] [src/maintainers/externalmaintainer.go:mainLoop:571] [00:31:44.064] Got a new descriptor for mongot [2021-10-22T00:31:44.064+0000] [.info] [src/maintainers/externalmaintainer.go:mainLoop:571] [00:31:44.064] Got a new descriptor for atlasProxy [2021-10-22T00:31:44.252+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:31:44.252] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:31:44.280+0000] [.info] [src/modules/maintainer.go:StartModule:315] [00:31:44.280] Started Backup Module [2021-10-22T00:31:44.289+0000] [.info] [src/director/director.go:planAndExecute:537] [00:31:44.289] In Goal State for clusterConfig version: 42 [2021-10-22T00:31:45.200+0000] [.info] [main/components/agent.go:Iterate:852] [00:31:45.200] All 1 Mongo processes are in goal state. CPS in goal state [2021-10-22T00:31:45.250+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:31:45.250] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:31:45.298+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:31:45.298] clusterConfig unchanged [2021-10-22T00:31:46.251+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:31:46.251] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:31:47.267+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:31:47.267] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:31:48.266+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:31:48.266] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:31:49.275+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:31:49.275] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:31:50.265+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:31:50.265] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:31:51.269+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:31:51.269] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:31:52.273+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:31:52.273] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:31:53.273+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:31:53.273] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:31:54.273+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:31:54.273] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:31:54.490+0000] [.info] [src/director/director.go:planAndExecute:537] [00:31:54.490] In Goal State for clusterConfig version: 42 [2021-10-22T00:31:55.278+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:31:55.278] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:31:55.311+0000] [.info] [main/components/agent.go:Iterate:852] [00:31:55.311] All 1 Mongo processes are in goal state. Backup in goal state, CPS in goal state [2021-10-22T00:31:55.966+0000] [.info] [src/config/config.go:ReadClusterConfig:509] [00:31:55.966] Backing up cluster config to /var/lib/mongodb-mms-automation/mms-cluster-config-backup.json [2021-10-22T00:31:55.967+0000] [.info] [main/components/agent.go:LoadClusterConfig:262] [00:31:55.967] New cluster config received! 42 ("692dea1cb47ae80ebf0db09c534b1de5--gzip") -> 43 ("a3c2386cf0a0d3acaa1599cc09ea92c3--gzip") [2021-10-22T00:31:56.276+0000] [.info] [metrics/collector/realftscollector.go:mainLoop:239] [00:31:56.276] Received new cluster config! [2021-10-22T00:31:56.276+0000] [.info] [metrics/prometheus/exporter.go:mainLoop:73] [00:31:56.276] Received new cluster config! [2021-10-22T00:31:56.276+0000] [.info] [metrics/prometheus/server.go:ApplyNewPromConfig:53] [00:31:56.276] Prometheus config is nil or has prometheus explicitly disabled, stopping the prometheus server if running... [2021-10-22T00:31:56.276+0000] [.info] [metrics/prometheus/server.go:func1:51] [00:31:56.276] ApplyNewPromConfig took 28.613µs [2021-10-22T00:31:56.277+0000] [.info] [realtime/rtcollector/rtcollector.go:controlLoop:95] [00:31:56.277] Received new cluster config! [2021-10-22T00:31:56.277+0000] [.info] [src/dataexplorer/dataexplorer.go:controlLoop:114] [00:31:56.277] Received new cluster config! [2021-10-22T00:31:56.277+0000] [.info] [src/filebeat/filebeat.go:controlLoop:634] [00:31:56.277] Received new cluster config! [2021-10-22T00:31:56.291+0000] [.info] [src/director/director.go:mainLoop:444] [00:31:56.291] Received new cluster config! [2021-10-22T00:31:56.291+0000] [.info] [src/director/director.go:incorporateConfig:302] [00:31:56.291] clusterConfig edition is different ("692dea1cb47ae80ebf0db09c534b1de5--gzip" -> "a3c2386cf0a0d3acaa1599cc09ea92c3--gzip"). Incorporating... [2021-10-22T00:31:56.298+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:31:56.298] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:31:56.299+0000] [.info] [metrics/collector/realcollector.go:collectLoop:253] [00:31:56.299] Received new cluster config! [2021-10-22T00:31:56.306+0000] [.info] [src/maintainers/externalmaintainer.go:mainLoop:571] [00:31:56.306] Got a new descriptor for mongot [2021-10-22T00:31:56.307+0000] [.info] [src/maintainers/externalmaintainer.go:mainLoop:571] [00:31:56.307] Got a new descriptor for atlasProxy [2021-10-22T00:31:56.693+0000] [.info] [src/director/director.go:planAndExecute:537] [00:31:56.693] In Goal State for clusterConfig version: 43 [2021-10-22T00:31:57.288+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:31:57.288] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:31:58.287+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:31:58.287] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:31:59.297+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:31:59.297] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:00.300+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:00.300] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:01.306+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:01.306] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:02.299+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:02.299] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:03.301+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:03.301] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:04.302+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:04.302] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:05.301+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:05.301] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:06.303+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:06.303] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:06.395+0000] [.info] [main/components/agent.go:Iterate:852] [00:32:06.395] All 1 Mongo processes are in goal state. Backup in goal state, CPS in goal state [2021-10-22T00:32:06.613+0000] [.info] [src/config/config.go:ReadClusterConfig:509] [00:32:06.613] Backing up cluster config to /var/lib/mongodb-mms-automation/mms-cluster-config-backup.json [2021-10-22T00:32:06.613+0000] [.info] [main/components/agent.go:LoadClusterConfig:262] [00:32:06.613] New cluster config received! 43 ("a3c2386cf0a0d3acaa1599cc09ea92c3--gzip") -> 44 ("5f7f9357066bab1c1f8c1ddbed332c73--gzip") [2021-10-22T00:32:06.822+0000] [.info] [metrics/collector/realcollector.go:collectLoop:253] [00:32:06.822] Received new cluster config! [2021-10-22T00:32:06.822+0000] [.info] [metrics/collector/realftscollector.go:mainLoop:239] [00:32:06.822] Received new cluster config! [2021-10-22T00:32:06.822+0000] [.info] [metrics/prometheus/exporter.go:mainLoop:73] [00:32:06.822] Received new cluster config! [2021-10-22T00:32:06.822+0000] [.info] [metrics/prometheus/server.go:ApplyNewPromConfig:53] [00:32:06.822] Prometheus config is nil or has prometheus explicitly disabled, stopping the prometheus server if running... [2021-10-22T00:32:06.822+0000] [.info] [metrics/prometheus/server.go:func1:51] [00:32:06.822] ApplyNewPromConfig took 32.18µs [2021-10-22T00:32:06.822+0000] [.info] [src/director/director.go:incorporateConfig:302] [00:32:06.822] clusterConfig edition is different ("a3c2386cf0a0d3acaa1599cc09ea92c3--gzip" -> "5f7f9357066bab1c1f8c1ddbed332c73--gzip"). Incorporating... [2021-10-22T00:32:06.846+0000] [.info] [realtime/rtcollector/rtcollector.go:controlLoop:95] [00:32:06.846] Received new cluster config! [2021-10-22T00:32:06.846+0000] [.info] [src/dataexplorer/dataexplorer.go:controlLoop:114] [00:32:06.846] Received new cluster config! [2021-10-22T00:32:06.846+0000] [.info] [src/filebeat/filebeat.go:controlLoop:634] [00:32:06.846] Received new cluster config! [2021-10-22T00:32:06.929+0000] [.info] [src/maintainers/externalmaintainer.go:mainLoop:571] [00:32:06.929] Got a new descriptor for mongot [2021-10-22T00:32:06.930+0000] [.info] [src/maintainers/externalmaintainer.go:mainLoop:571] [00:32:06.930] Got a new descriptor for atlasProxy [2021-10-22T00:32:07.127+0000] [.info] [src/director/director.go:planAndExecute:537] [00:32:07.127] In Goal State for clusterConfig version: 44 [2021-10-22T00:32:07.304+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:07.304] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:08.302+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:08.302] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:09.302+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:09.302] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:10.304+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:10.304] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:11.238+0000] [.info] [src/mmsfeatures/mmsfeatures.go:postLoop:57] [00:32:11.238] Updating agent supported features. [2021-10-22T00:32:11.308+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:11.308] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:11.370+0000] [.info] [src/mmsfeatures/mmsfeatures.go:postLoop:64] [00:32:11.370] Agent features sent; stopping. [2021-10-22T00:32:12.308+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:12.308] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:13.306+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:13.306] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:14.306+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:14.306] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:15.307+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:15.307] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:16.308+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:16.308] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:17.013+0000] [.info] [main/components/agent.go:Iterate:852] [00:32:17.013] All 1 Mongo processes are in goal state. Backup in goal state, CPS in goal state [2021-10-22T00:32:17.111+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:32:17.111] clusterConfig unchanged [2021-10-22T00:32:17.248+0000] [.info] [src/director/director.go:planAndExecute:537] [00:32:17.248] In Goal State for clusterConfig version: 44 [2021-10-22T00:32:17.308+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:17.308] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:18.308+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:18.308] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:19.309+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:19.309] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:20.309+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:20.309] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:21.310+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:21.310] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:22.315+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:22.315] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:23.311+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:23.311] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:24.315+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:24.315] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:25.313+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:25.313] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:26.315+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:26.315] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:27.112+0000] [.info] [main/components/agent.go:Iterate:852] [00:32:27.112] All 1 Mongo processes are in goal state. Backup in goal state, CPS in goal state [2021-10-22T00:32:27.326+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:27.326] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:27.361+0000] [.info] [src/config/config.go:ReadClusterConfig:509] [00:32:27.361] Backing up cluster config to /var/lib/mongodb-mms-automation/mms-cluster-config-backup.json [2021-10-22T00:32:27.361+0000] [.info] [main/components/agent.go:LoadClusterConfig:262] [00:32:27.361] New cluster config received! 44 ("5f7f9357066bab1c1f8c1ddbed332c73--gzip") -> 51 ("973422c9462ea20cccc0685177661856--gzip") [2021-10-22T00:32:27.420+0000] [.info] [src/director/director.go:planAndExecute:537] [00:32:27.420] In Goal State for clusterConfig version: 44 [2021-10-22T00:32:27.527+0000] [.info] [src/director/director.go:mainLoop:444] [00:32:27.527] Received new cluster config! [2021-10-22T00:32:27.527+0000] [.info] [metrics/collector/realcollector.go:collectLoop:253] [00:32:27.527] Received new cluster config! [2021-10-22T00:32:27.527+0000] [.info] [metrics/collector/realftscollector.go:mainLoop:239] [00:32:27.527] Received new cluster config! [2021-10-22T00:32:27.527+0000] [.info] [metrics/prometheus/exporter.go:mainLoop:73] [00:32:27.527] Received new cluster config! [2021-10-22T00:32:27.527+0000] [.info] [metrics/prometheus/server.go:ApplyNewPromConfig:53] [00:32:27.527] Prometheus config is nil or has prometheus explicitly disabled, stopping the prometheus server if running... [2021-10-22T00:32:27.527+0000] [.info] [metrics/prometheus/server.go:func1:51] [00:32:27.527] ApplyNewPromConfig took 19.316µs [2021-10-22T00:32:27.528+0000] [.info] [realtime/rtcollector/rtcollector.go:controlLoop:95] [00:32:27.528] Received new cluster config! [2021-10-22T00:32:27.528+0000] [.info] [src/dataexplorer/dataexplorer.go:controlLoop:114] [00:32:27.528] Received new cluster config! [2021-10-22T00:32:27.528+0000] [.info] [src/filebeat/filebeat.go:controlLoop:634] [00:32:27.528] Received new cluster config! [2021-10-22T00:32:27.530+0000] [.info] [src/director/director.go:incorporateConfig:302] [00:32:27.530] clusterConfig edition is different ("5f7f9357066bab1c1f8c1ddbed332c73--gzip" -> "973422c9462ea20cccc0685177661856--gzip"). Incorporating... [2021-10-22T00:32:27.549+0000] [.info] [src/filebeat/filebeat.go:stop:784] [00:32:27.549] Stopping filebeat log collection... [2021-10-22T00:32:27.549+0000] [.info] [src/filebeat/filebeat.go:stop:789] [00:32:27.549] Filebeat log collection stopped. [2021-10-22T00:32:27.549+0000] [.info] [src/filebeat/filebeat.go:run:666] [00:32:27.549] Starting filebeat log tailing [2021-10-22T00:32:27.552+0000] [.info] [filebeat/mspooler/mspooler.go:Start:65] [00:32:27.552] ClientMetadata Starting mspooler loop to listen for all log events [2021-10-22T00:32:27.552+0000] [.info] [filebeat/mspooler/mspooler.go:Start:65] [00:32:27.552] SlowQueries Starting mspooler loop to listen for all log events [2021-10-22T00:32:27.552+0000] [.info] [filebeat/mspooler/mspooler.go:Start:65] [00:32:27.552] AccessLogs Starting mspooler loop to listen for all log events [2021-10-22T00:32:27.552+0000] [.info] [filebeat/mspooler/mspooler.go:Start:65] [00:32:27.552] LogIngestion Starting mspooler loop to listen for all log events [2021-10-22T00:32:27.552+0000] [.info] [filebeat/mspooler/regSpooler.go:Start:40] [00:32:27.552] Starting registrarSpooler loop to listen for all log events [2021-10-22T00:32:27.554+0000] [.info] [mspooler/logingestion/logingestion.go:periodicallyRequestRules:98] [00:32:27.554] LogIngestion making rules request [2021-10-22T00:32:27.696+0000] [.info] [mspooler/logingestion/logingestion.go:periodicallyRequestRules:112] [00:32:27.696] LogIngestion received rules: [] [2021-10-22T00:32:28.319+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:28.319] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:29.314+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:29.314] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:30.315+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:30.314] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:31.315+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:31.315] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:32.316+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:32.316] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:32.573+0000] [.info] [src/maintainers/externalmaintainer.go:mainLoop:571] [00:32:32.572] Got a new descriptor for mongot [2021-10-22T00:32:32.573+0000] [.info] [src/maintainers/externalmaintainer.go:mainLoop:571] [00:32:32.572] Got a new descriptor for atlasProxy [2021-10-22T00:32:32.740+0000] [.info] [src/director/director.go:computePlan:280] [00:32:32.740] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:32:32.740+0000] [.info] [src/director/director.go:tracef:794] [00:32:32.740] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:32:32.740+0000] [.info] [src/director/director.go:tracef:794] [00:32:32.740] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = -1 is <= phase = 0' = true] ] [2021-10-22T00:32:32.740+0000] [.info] [src/director/director.go:tracef:794] [00:32:32.740] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:32:32.740+0000] [.info] [src/director/director.go:tracef:794] [00:32:32.740] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:32:32.742+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:32:32.742] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:32:32.887+0000] [.info] [src/director/director.go:tracef:794] [00:32:32.887] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:32:32.887+0000] [.info] [src/director/director.go:tracef:794] [00:32:32.887] because [2021-10-22T00:32:32.887+0000] [.info] [src/director/director.go:tracef:794] [00:32:32.887] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:32:32.887+0000] [.info] [src/director/director.go:tracef:794] [00:32:32.887] because ['currentState.CanBounceStop' = true] [2021-10-22T00:32:33.019+0000] [.info] [src/director/director.go:tracef:794] [00:32:33.019] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:32:33.019+0000] [.info] [src/director/director.go:tracef:794] [00:32:33.019] because [2021-10-22T00:32:33.020+0000] [.info] [src/director/director.go:tracef:794] [00:32:33.020] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:32:33.020+0000] [.info] [src/director/director.go:tracef:794] [00:32:33.020] because ['currentState.Up' = false] [2021-10-22T00:32:33.020+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:203] [00:32:33.020] Found matching directory name : /srv/mongodb/atlas-b7bb9t-config-0-node-0 [2021-10-22T00:32:33.020+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:32:33.020] SourceDbPath: /srv/mongodb/atlas-b7bb9t-config-0-node-0 TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:32:33.020+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:220] [00:32:33.020] Deleting the original TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:32:33.020+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:224] [00:32:33.020] Deleted the original TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:32:33.020+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:226] [00:32:33.020] Renaming directory from /srv/mongodb/atlas-b7bb9t-config-0-node-0 to /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:32:33.020+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:230] [00:32:33.020] Renamed directory from /srv/mongodb/atlas-b7bb9t-config-0-node-0 to /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:32:33.104+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:32:33.104] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:32:33.108+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:32:33.108] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:32:33.109+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:32:33.109] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:32:33.111+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:32:33.111] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:32:33.111+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:32:33.111] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:32:33.111+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:32:33.111] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:32:33.114+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:32:33.114] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:32:33.286+0000] [.info] [src/director/director.go:tracef:794] [00:32:33.286] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:32:33.286+0000] [.info] [src/director/director.go:tracef:794] [00:32:33.286] because ['currentState.Up' = false] [2021-10-22T00:32:33.332+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [00:32:33.332] lsblk cache invalidated, running lsblk command [2021-10-22T00:32:33.335+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:33.335] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:33.823+0000] [.info] [src/director/director.go:tracef:794] [00:32:33.823] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:32:33.823+0000] [.info] [src/director/director.go:tracef:794] [00:32:33.823] because ['currentState.Up' = false] [2021-10-22T00:32:33.823+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:32:33.823] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:32:34.108+0000] [.info] [src/director/director.go:tracef:794] [00:32:34.108] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:32:34.108+0000] [.info] [src/director/director.go:tracef:794] [00:32:34.108] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:32:34.322+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:34.322] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:34.338+0000] [.info] [src/director/director.go:tracef:794] [00:32:34.338] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:32:34.338+0000] [.info] [src/director/director.go:tracef:794] [00:32:34.338] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:32:34.339+0000] [.info] [src/director/director.go:tracef:794] [00:32:34.339] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:32:34.339+0000] [.info] [src/director/director.go:tracef:794] [00:32:34.339] because ['currentState.Up' = false] [2021-10-22T00:32:34.339+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:32:34.339] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:32:34.339+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:32:34.339] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:32:34.375+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:32:34.375] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:32:34.376+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:32:34.376] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48663 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:32:35.320+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:35.320] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:35.737+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:32:35.737] ...Finished start [pid=9682] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48663,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:32:35.809+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:32:35.809] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 71.2665ms. [2021-10-22T00:32:35.809+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:32:35.809] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:48663 (local=false) [2021-10-22T00:32:35.809+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:32:35.809] Dropping existing documents from system.replset [2021-10-22T00:32:35.869+0000] [.info] [src/mongoctl/processctl.go:RemoveAllIfExists:3360] [00:32:35.869] Removed document(s) using query map[] from local.system.replset [2021-10-22T00:32:35.869+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:32:35.869] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:32:35.875+0000] [.info] [src/mongoctl/processctl.go:RemoveAllIfExists:3360] [00:32:35.875] Removed document(s) using query map[] from local.replset.oplogTruncateAfterPoint [2021-10-22T00:32:35.882+0000] [.info] [src/mongoctl/processctl.go:RemoveAllIfExists:3360] [00:32:35.882] Removed document(s) using query map[] from local.replset.minvalid [2021-10-22T00:32:35.889+0000] [.info] [src/mongoctl/processctl.go:RemoveAllIfExists:3360] [00:32:35.889] Removed document(s) using query map[] from local.replset.election [2021-10-22T00:32:35.897+0000] [.info] [src/mongoctl/processctl.go:RemoveAllIfExists:3360] [00:32:35.897] Removed document(s) using query map[] from local.replset.initialSyncId [2021-10-22T00:32:35.897+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:32:35.897] Inserting oplogTruncateAfterPoint document [2021-10-22T00:32:35.897+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:32:35.897] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:32:35.899+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:32:35.899] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:32:35.900+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:32:35.900] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:32:35.900+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:32:35.900] Setting the minvalid document to [{_id ObjectID("617206a3523370002becc19b")} {t -1} {ts {0 1}}] [2021-10-22T00:32:35.901+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:32:35.901] Inserted [{_id ObjectID("617206a3523370002becc19b")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:32:35.901+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:32:35.901] Inserting new replica set configuration document. [2021-10-22T00:32:35.901+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:32:35.901] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:32:35.903+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:32:35.903] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:32:35.908+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:32:35.908] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:32:35.908+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:32:35.908] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:32:35.908+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:32:35.908] Initiating stopEphemeralMongo [2021-10-22T00:32:35.908+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:32:35.908] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:48663 (local=false) [2021-10-22T00:32:35.908+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:32:35.908] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:48663 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:32:36.101+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:32:36.101] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.698085ms. [2021-10-22T00:32:36.101+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:32:36.101] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:48663 (local=false) [2021-10-22T00:32:36.101+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:32:36.101] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 29.02µs. [2021-10-22T00:32:36.101+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:32:36.101] Successfully finished running stopEphemeralMongo [2021-10-22T00:32:36.101+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:32:36.101] Successfully stopped existing ephemeral mongod. [2021-10-22T00:32:36.101+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:32:36.101] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:32:36.101+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:32:36.101] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:32:36.134+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:32:36.134] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:32:36.135+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:32:36.135] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41503 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:32:36.321+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:36.321] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:37.321+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:37.321] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:37.328+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:32:37.328] ...Finished start [pid=11632] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41503,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:32:37.379+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:32:37.379] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 51.193635ms. [2021-10-22T00:32:37.379+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1613] [00:32:37.379] Successfully started up a standalone mongod on an ephemeral port with oplog replay params. [2021-10-22T00:32:37.379+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:32:37.379] Initiating stopEphemeralMongo [2021-10-22T00:32:37.379+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:32:37.379] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41503 (local=false) [2021-10-22T00:32:37.379+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:32:37.379] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41503 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:32:37.567+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:32:37.567] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.93212ms. [2021-10-22T00:32:37.567+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:32:37.567] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41503 (local=false) [2021-10-22T00:32:37.567+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:32:37.567] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.539µs. [2021-10-22T00:32:37.567+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:32:37.567] Successfully finished running stopEphemeralMongo [2021-10-22T00:32:37.567+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1618] [00:32:37.567] Successfully shut down a standalone mongod on an ephemeral port with oplog replay params. [2021-10-22T00:32:37.567+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:32:37.567] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:32:37.567+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:32:37.567] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:32:37.631+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:32:37.631] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:32:37.632+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:32:37.632] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36897 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:32:38.331+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:38.331] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:38.896+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:32:38.896] ...Finished start [pid=13211] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36897,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:32:38.950+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:32:38.950] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 53.402085ms. [2021-10-22T00:32:38.950+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:32:38.950] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36897 (local=false) [2021-10-22T00:32:38.950+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1623] [00:32:38.950] Successfully started up a standalone mongod on an ephemeral port [2021-10-22T00:32:38.950+0000] [.info] [src/action/restorefrombackup.go:checkAlreadyHasOplog:1238] [00:32:38.950] Starting checkAlreadyHasOplog [2021-10-22T00:32:38.996+0000] [.info] [src/action/restorefrombackup.go:updateShardInfoForRestore:1628] [00:32:38.996] Updating shard metadata for restore [2021-10-22T00:32:38.997+0000] [.info] [src/action/restorefrombackup.go:updateShardMetadataForRestore:2016] [00:32:38.997] Setting balancer doc to stopped:false in config.settings [2021-10-22T00:32:39.013+0000] [.info] [src/mongoctl/processctl.go:Update:3279] [00:32:39.013] Updated with query map[_id:balancer] and update map[$set:map[stopped:false]] and upsert=true on config.settings [2021-10-22T00:32:39.013+0000] [.info] [src/action/restorefrombackup.go:UpdateShardMetadata:2034] [00:32:39.013] Removing config.mongos collection [2021-10-22T00:32:39.024+0000] [.info] [src/mongoctl/processctl.go:RemoveAllIfExists:3360] [00:32:39.024] Removed document(s) using query map[] from config.mongos [2021-10-22T00:32:39.024+0000] [.info] [src/action/restorefrombackup.go:UpdateShardMetadata:2038] [00:32:39.024] Removing config.lockpings collection [2021-10-22T00:32:39.037+0000] [.info] [src/mongoctl/processctl.go:RemoveAllIfExists:3360] [00:32:39.037] Removed document(s) using query map[] from config.lockpings [2021-10-22T00:32:39.037+0000] [.info] [src/action/restorefrombackup.go:UpdateShardMetadata:2053] [00:32:39.037] Updating shard metadata on config server. [2021-10-22T00:32:39.043+0000] [.info] [src/action/restorefrombackup.go:UpdateShardMetadata:2070] [00:32:39.043] Found expected number of shards (2). Matching those against 2 custom shard mappings. [2021-10-22T00:32:39.043+0000] [.info] [src/action/restorefrombackup.go:UpdateShardMetadata:2071] [00:32:39.043] Custom shard mapping [&{SourceShardId:atlas-b7bb9t-shard-0 TargetShardId:atlas-4wnh1p-shard-0}, &{SourceShardId:atlas-b7bb9t-shard-1 TargetShardId:atlas-4wnh1p-shard-1}]. [2021-10-22T00:32:39.043+0000] [.info] [src/action/restorefrombackup.go:UpdateShardMetadata:2111] [00:32:39.043] shardMapping 0 -> 0: {map[_id:atlas-b7bb9t-shard-0 host:atlas-b7bb9t-shard-0/atlas-b7bb9t-shard-00-00.qzt8l.mmscloudteam.com:27017,atlas-b7bb9t-shard-00-01.qzt8l.mmscloudteam.com:27017,atlas-b7bb9t-shard-00-02.qzt8l.mmscloudteam.com:27017 state:1] atlas-b7bb9t-shard-0 atlas-4wnh1p-shard-0 atlas-4wnh1p-shard-0 [atlas-4wnh1p-shard-00-00.qzt8l.mmscloudteam.com:27017 atlas-4wnh1p-shard-00-01.qzt8l.mmscloudteam.com:27017 atlas-4wnh1p-shard-00-02.qzt8l.mmscloudteam.com:27017]} [2021-10-22T00:32:39.043+0000] [.info] [src/action/restorefrombackup.go:UpdateShardMetadata:2111] [00:32:39.043] shardMapping 1 -> 1: {map[_id:atlas-b7bb9t-shard-1 host:atlas-b7bb9t-shard-1/atlas-b7bb9t-shard-01-00.qzt8l.mmscloudteam.com:27017,atlas-b7bb9t-shard-01-01.qzt8l.mmscloudteam.com:27017,atlas-b7bb9t-shard-01-02.qzt8l.mmscloudteam.com:27017 state:1] atlas-b7bb9t-shard-1 atlas-4wnh1p-shard-1 atlas-4wnh1p-shard-1 [atlas-4wnh1p-shard-01-00.qzt8l.mmscloudteam.com:27017 atlas-4wnh1p-shard-01-01.qzt8l.mmscloudteam.com:27017 atlas-4wnh1p-shard-01-02.qzt8l.mmscloudteam.com:27017]} [2021-10-22T00:32:39.043+0000] [.info] [src/action/restorefrombackup.go:UpdateShardMetadata:2114] [00:32:39.043] Final shardMapping [{shardMetadata:map[_id:atlas-b7bb9t-shard-0 host:atlas-b7bb9t-shard-0/atlas-b7bb9t-shard-00-00.qzt8l.mmscloudteam.com:27017,atlas-b7bb9t-shard-00-01.qzt8l.mmscloudteam.com:27017,atlas-b7bb9t-shard-00-02.qzt8l.mmscloudteam.com:27017 state:1] oldShardId:atlas-b7bb9t-shard-0 newShardId:atlas-4wnh1p-shard-0 newShardRsId:atlas-4wnh1p-shard-0 newShardRsHostPorts:[atlas-4wnh1p-shard-00-00.qzt8l.mmscloudteam.com:27017 atlas-4wnh1p-shard-00-01.qzt8l.mmscloudteam.com:27017 atlas-4wnh1p-shard-00-02.qzt8l.mmscloudteam.com:27017]} {shardMetadata:map[_id:atlas-b7bb9t-shard-1 host:atlas-b7bb9t-shard-1/atlas-b7bb9t-shard-01-00.qzt8l.mmscloudteam.com:27017,atlas-b7bb9t-shard-01-01.qzt8l.mmscloudteam.com:27017,atlas-b7bb9t-shard-01-02.qzt8l.mmscloudteam.com:27017 state:1] oldShardId:atlas-b7bb9t-shard-1 newShardId:atlas-4wnh1p-shard-1 newShardRsId:atlas-4wnh1p-shard-1 newShardRsHostPorts:[atlas-4wnh1p-shard-01-00.qzt8l.mmscloudteam.com:27017 atlas-4wnh1p-shard-01-01.qzt8l.mmscloudteam.com:27017 atlas-4wnh1p-shard-01-02.qzt8l.mmscloudteam.com:27017]}] [2021-10-22T00:32:39.043+0000] [.info] [src/action/restorefrombackup.go:UpdateShardMetadata:2116] [00:32:39.043] Pass 1: Renaming old shard ids to temp names [2021-10-22T00:32:39.043+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2209] [00:32:39.043] Updating config.databases db : updating documents with {'primary' : atlas-b7bb9t-shard-0} to {'primary' : atlas-4wnh1p-shard-0.temp} [2021-10-22T00:32:39.049+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.049] Updated all with query [{primary atlas-b7bb9t-shard-0}] and update [{$set [{primary atlas-4wnh1p-shard-0.temp}]}] on config.databases [2021-10-22T00:32:39.049+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2216] [00:32:39.049] Updating config.chunks db : updating documents with {'shard' : atlas-b7bb9t-shard-0} to {'shard' : atlas-4wnh1p-shard-0.temp} [2021-10-22T00:32:39.086+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.086] Updated all with query [{shard atlas-b7bb9t-shard-0}] and update [{$set [{shard atlas-4wnh1p-shard-0.temp} {history []}]}] on config.chunks [2021-10-22T00:32:39.086+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2227] [00:32:39.086] Updating config.collections db : updating documents with {'primary' : atlas-b7bb9t-shard-0} to {'primary' : atlas-4wnh1p-shard-0.temp} [2021-10-22T00:32:39.092+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.092] Updated all with query [{primary atlas-b7bb9t-shard-0}] and update [{$set [{primary atlas-4wnh1p-shard-0.temp}]}] on config.collections [2021-10-22T00:32:39.092+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2243] [00:32:39.092] Updating config.shards db : removing a document with {'_id' : atlas-b7bb9t-shard-0} and inserting a document {'_id' : atlas-4wnh1p-shard-0.temp, 'host' : atlas-4wnh1p-shard-0.temp/atlas-4wnh1p-shard-00-00.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-00-01.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-00-02.qzt8l.mmscloudteam.com:27017} [2021-10-22T00:32:39.097+0000] [.info] [src/mongoctl/processctl.go:Remove:3319] [00:32:39.097] Removed document using query [{_id atlas-b7bb9t-shard-0}] from config.shards [2021-10-22T00:32:39.097+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2275] [00:32:39.097] Upserting config.shards { _id: atlas-4wnh1p-shard-0.temp } with [{_id atlas-4wnh1p-shard-0.temp} {host atlas-4wnh1p-shard-0.temp/atlas-4wnh1p-shard-00-00.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-00-01.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-00-02.qzt8l.mmscloudteam.com:27017} {state 1}] [2021-10-22T00:32:39.100+0000] [.info] [src/mongoctl/processctl.go:ReplaceOne:3243] [00:32:39.100] Replaced with query map[_id:atlas-4wnh1p-shard-0.temp] and replacement [{_id atlas-4wnh1p-shard-0.temp} {host atlas-4wnh1p-shard-0.temp/atlas-4wnh1p-shard-00-00.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-00-01.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-00-02.qzt8l.mmscloudteam.com:27017} {state 1}] and upsert=true on config.shards [2021-10-22T00:32:39.100+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2291] [00:32:39.100] Successfully updated config metadata for atlas-b7bb9t-shard-0 -> atlas-4wnh1p-shard-0.temp [2021-10-22T00:32:39.100+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2209] [00:32:39.100] Updating config.databases db : updating documents with {'primary' : atlas-b7bb9t-shard-1} to {'primary' : atlas-4wnh1p-shard-1.temp} [2021-10-22T00:32:39.102+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.102] Updated all with query [{primary atlas-b7bb9t-shard-1}] and update [{$set [{primary atlas-4wnh1p-shard-1.temp}]}] on config.databases [2021-10-22T00:32:39.102+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2216] [00:32:39.102] Updating config.chunks db : updating documents with {'shard' : atlas-b7bb9t-shard-1} to {'shard' : atlas-4wnh1p-shard-1.temp} [2021-10-22T00:32:39.148+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.148] Updated all with query [{shard atlas-b7bb9t-shard-1}] and update [{$set [{shard atlas-4wnh1p-shard-1.temp} {history []}]}] on config.chunks [2021-10-22T00:32:39.148+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2227] [00:32:39.148] Updating config.collections db : updating documents with {'primary' : atlas-b7bb9t-shard-1} to {'primary' : atlas-4wnh1p-shard-1.temp} [2021-10-22T00:32:39.149+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.149] Updated all with query [{primary atlas-b7bb9t-shard-1}] and update [{$set [{primary atlas-4wnh1p-shard-1.temp}]}] on config.collections [2021-10-22T00:32:39.150+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2243] [00:32:39.150] Updating config.shards db : removing a document with {'_id' : atlas-b7bb9t-shard-1} and inserting a document {'_id' : atlas-4wnh1p-shard-1.temp, 'host' : atlas-4wnh1p-shard-1.temp/atlas-4wnh1p-shard-01-00.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-01-01.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-01-02.qzt8l.mmscloudteam.com:27017} [2021-10-22T00:32:39.151+0000] [.info] [src/mongoctl/processctl.go:Remove:3319] [00:32:39.151] Removed document using query [{_id atlas-b7bb9t-shard-1}] from config.shards [2021-10-22T00:32:39.151+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2275] [00:32:39.151] Upserting config.shards { _id: atlas-4wnh1p-shard-1.temp } with [{_id atlas-4wnh1p-shard-1.temp} {host atlas-4wnh1p-shard-1.temp/atlas-4wnh1p-shard-01-00.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-01-01.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-01-02.qzt8l.mmscloudteam.com:27017} {state 1}] [2021-10-22T00:32:39.154+0000] [.info] [src/mongoctl/processctl.go:ReplaceOne:3243] [00:32:39.154] Replaced with query map[_id:atlas-4wnh1p-shard-1.temp] and replacement [{_id atlas-4wnh1p-shard-1.temp} {host atlas-4wnh1p-shard-1.temp/atlas-4wnh1p-shard-01-00.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-01-01.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-01-02.qzt8l.mmscloudteam.com:27017} {state 1}] and upsert=true on config.shards [2021-10-22T00:32:39.154+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2291] [00:32:39.154] Successfully updated config metadata for atlas-b7bb9t-shard-1 -> atlas-4wnh1p-shard-1.temp [2021-10-22T00:32:39.154+0000] [.info] [src/action/restorefrombackup.go:UpdateShardMetadata:2140] [00:32:39.154] Pass 2: Renaming temp names to new shard ids [2021-10-22T00:32:39.154+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2209] [00:32:39.154] Updating config.databases db : updating documents with {'primary' : atlas-4wnh1p-shard-0.temp} to {'primary' : atlas-4wnh1p-shard-0} [2021-10-22T00:32:39.157+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.157] Updated all with query [{primary atlas-4wnh1p-shard-0.temp}] and update [{$set [{primary atlas-4wnh1p-shard-0}]}] on config.databases [2021-10-22T00:32:39.157+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2216] [00:32:39.157] Updating config.chunks db : updating documents with {'shard' : atlas-4wnh1p-shard-0.temp} to {'shard' : atlas-4wnh1p-shard-0} [2021-10-22T00:32:39.187+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.187] Updated all with query [{shard atlas-4wnh1p-shard-0.temp}] and update [{$set [{shard atlas-4wnh1p-shard-0} {history []}]}] on config.chunks [2021-10-22T00:32:39.187+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2227] [00:32:39.187] Updating config.collections db : updating documents with {'primary' : atlas-4wnh1p-shard-0.temp} to {'primary' : atlas-4wnh1p-shard-0} [2021-10-22T00:32:39.189+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.189] Updated all with query [{primary atlas-4wnh1p-shard-0.temp}] and update [{$set [{primary atlas-4wnh1p-shard-0}]}] on config.collections [2021-10-22T00:32:39.189+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2243] [00:32:39.189] Updating config.shards db : removing a document with {'_id' : atlas-4wnh1p-shard-0.temp} and inserting a document {'_id' : atlas-4wnh1p-shard-0, 'host' : atlas-4wnh1p-shard-0/atlas-4wnh1p-shard-00-00.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-00-01.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-00-02.qzt8l.mmscloudteam.com:27017} [2021-10-22T00:32:39.190+0000] [.info] [src/mongoctl/processctl.go:Remove:3319] [00:32:39.190] Removed document using query [{_id atlas-4wnh1p-shard-0.temp}] from config.shards [2021-10-22T00:32:39.191+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2275] [00:32:39.190] Upserting config.shards { _id: atlas-4wnh1p-shard-0 } with [{_id atlas-4wnh1p-shard-0} {host atlas-4wnh1p-shard-0/atlas-4wnh1p-shard-00-00.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-00-01.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-00-02.qzt8l.mmscloudteam.com:27017} {state 1}] [2021-10-22T00:32:39.194+0000] [.info] [src/mongoctl/processctl.go:ReplaceOne:3243] [00:32:39.194] Replaced with query map[_id:atlas-4wnh1p-shard-0] and replacement [{_id atlas-4wnh1p-shard-0} {host atlas-4wnh1p-shard-0/atlas-4wnh1p-shard-00-00.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-00-01.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-00-02.qzt8l.mmscloudteam.com:27017} {state 1}] and upsert=true on config.shards [2021-10-22T00:32:39.194+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2291] [00:32:39.194] Successfully updated config metadata for atlas-4wnh1p-shard-0.temp -> atlas-4wnh1p-shard-0 [2021-10-22T00:32:39.194+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2209] [00:32:39.194] Updating config.databases db : updating documents with {'primary' : atlas-4wnh1p-shard-1.temp} to {'primary' : atlas-4wnh1p-shard-1} [2021-10-22T00:32:39.196+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.196] Updated all with query [{primary atlas-4wnh1p-shard-1.temp}] and update [{$set [{primary atlas-4wnh1p-shard-1}]}] on config.databases [2021-10-22T00:32:39.196+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2216] [00:32:39.196] Updating config.chunks db : updating documents with {'shard' : atlas-4wnh1p-shard-1.temp} to {'shard' : atlas-4wnh1p-shard-1} [2021-10-22T00:32:39.223+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.223] Updated all with query [{shard atlas-4wnh1p-shard-1.temp}] and update [{$set [{shard atlas-4wnh1p-shard-1} {history []}]}] on config.chunks [2021-10-22T00:32:39.223+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2227] [00:32:39.223] Updating config.collections db : updating documents with {'primary' : atlas-4wnh1p-shard-1.temp} to {'primary' : atlas-4wnh1p-shard-1} [2021-10-22T00:32:39.226+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.225] Updated all with query [{primary atlas-4wnh1p-shard-1.temp}] and update [{$set [{primary atlas-4wnh1p-shard-1}]}] on config.collections [2021-10-22T00:32:39.226+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2243] [00:32:39.226] Updating config.shards db : removing a document with {'_id' : atlas-4wnh1p-shard-1.temp} and inserting a document {'_id' : atlas-4wnh1p-shard-1, 'host' : atlas-4wnh1p-shard-1/atlas-4wnh1p-shard-01-00.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-01-01.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-01-02.qzt8l.mmscloudteam.com:27017} [2021-10-22T00:32:39.228+0000] [.info] [src/mongoctl/processctl.go:Remove:3319] [00:32:39.228] Removed document using query [{_id atlas-4wnh1p-shard-1.temp}] from config.shards [2021-10-22T00:32:39.228+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2275] [00:32:39.228] Upserting config.shards { _id: atlas-4wnh1p-shard-1 } with [{_id atlas-4wnh1p-shard-1} {host atlas-4wnh1p-shard-1/atlas-4wnh1p-shard-01-00.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-01-01.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-01-02.qzt8l.mmscloudteam.com:27017} {state 1}] [2021-10-22T00:32:39.231+0000] [.info] [src/mongoctl/processctl.go:ReplaceOne:3243] [00:32:39.231] Replaced with query map[_id:atlas-4wnh1p-shard-1] and replacement [{_id atlas-4wnh1p-shard-1} {host atlas-4wnh1p-shard-1/atlas-4wnh1p-shard-01-00.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-01-01.qzt8l.mmscloudteam.com:27017,atlas-4wnh1p-shard-01-02.qzt8l.mmscloudteam.com:27017} {state 1}] and upsert=true on config.shards [2021-10-22T00:32:39.231+0000] [.info] [src/action/restorefrombackup.go:updateIndividualShardMetadata:2291] [00:32:39.231] Successfully updated config metadata for atlas-4wnh1p-shard-1.temp -> atlas-4wnh1p-shard-1 [2021-10-22T00:32:39.231+0000] [.info] [src/action/restorefrombackup.go:UpdateShardMetadata:2158] [00:32:39.231] Successfully updated all config metadata. [2021-10-22T00:32:39.231+0000] [.info] [src/action/restorefrombackup.go:fixShardNamesInReshardingOperationsCollections:1752] [00:32:39.231] Updating documents in config.reshardingOperations to temporary names [2021-10-22T00:32:39.235+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.235] Updated all with query [{donorShards.id [{$eq atlas-b7bb9t-shard-0}]}] and update [{$set [{donorShards.$[elem].id atlas-4wnh1p-shard-0.temp}]}] on config.reshardingOperations [2021-10-22T00:32:39.237+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.237] Updated all with query [{recipientShards.id [{$eq atlas-b7bb9t-shard-0}]}] and update [{$set [{recipientShards.$[elem].id atlas-4wnh1p-shard-0.temp}]}] on config.reshardingOperations [2021-10-22T00:32:39.239+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.239] Updated all with query [{donorShards.id [{$eq atlas-b7bb9t-shard-1}]}] and update [{$set [{donorShards.$[elem].id atlas-4wnh1p-shard-1.temp}]}] on config.reshardingOperations [2021-10-22T00:32:39.242+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.242] Updated all with query [{recipientShards.id [{$eq atlas-b7bb9t-shard-1}]}] and update [{$set [{recipientShards.$[elem].id atlas-4wnh1p-shard-1.temp}]}] on config.reshardingOperations [2021-10-22T00:32:39.242+0000] [.info] [src/action/restorefrombackup.go:fixShardNamesInReshardingOperationsCollections:1759] [00:32:39.242] Updating documents in config.reshardingOperations from temporary names to final names [2021-10-22T00:32:39.245+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.244] Updated all with query [{donorShards.id [{$eq atlas-4wnh1p-shard-0.temp}]}] and update [{$set [{donorShards.$[elem].id atlas-4wnh1p-shard-0}]}] on config.reshardingOperations [2021-10-22T00:32:39.247+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.247] Updated all with query [{recipientShards.id [{$eq atlas-4wnh1p-shard-0.temp}]}] and update [{$set [{recipientShards.$[elem].id atlas-4wnh1p-shard-0}]}] on config.reshardingOperations [2021-10-22T00:32:39.249+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.249] Updated all with query [{donorShards.id [{$eq atlas-4wnh1p-shard-1.temp}]}] and update [{$set [{donorShards.$[elem].id atlas-4wnh1p-shard-1}]}] on config.reshardingOperations [2021-10-22T00:32:39.251+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.251] Updated all with query [{recipientShards.id [{$eq atlas-4wnh1p-shard-1.temp}]}] and update [{$set [{recipientShards.$[elem].id atlas-4wnh1p-shard-1}]}] on config.reshardingOperations [2021-10-22T00:32:39.251+0000] [.info] [src/action/restorefrombackup.go:fixShardNamesInReshardingOperationsCollections:1767] [00:32:39.251] Updating state in config.reshardingOperations [2021-10-22T00:32:39.253+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.253] Updated all with query [{state [{$ne committing}]}] and update [{$set [{state aborting} {abortReason [{code 341} {errmsg aborted by automation restore}]}]}] on config.reshardingOperations [2021-10-22T00:32:39.253+0000] [.info] [src/action/restorefrombackup.go:fixShardNamesInReshardingOperationsCollections:1772] [00:32:39.253] Successfully updating documents in config.reshardingOperations [2021-10-22T00:32:39.253+0000] [.info] [src/action/restorefrombackup.go:updateShardNameInTransactionCoordinatorsCollection:1873] [00:32:39.253] updating config.transaction_coordinators participants : replacing atlas-b7bb9t-shard-0 with atlas-4wnh1p-shard-0.temp [2021-10-22T00:32:39.254+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.254] Updated all with query [{participants atlas-b7bb9t-shard-0}] and update [{$set [{participants.$ atlas-4wnh1p-shard-0.temp}]}] on config.transaction_coordinators [2021-10-22T00:32:39.254+0000] [.info] [src/action/restorefrombackup.go:updateShardNameInTransactionCoordinatorsCollection:1873] [00:32:39.254] updating config.transaction_coordinators participants : replacing atlas-b7bb9t-shard-1 with atlas-4wnh1p-shard-1.temp [2021-10-22T00:32:39.255+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.255] Updated all with query [{participants atlas-b7bb9t-shard-1}] and update [{$set [{participants.$ atlas-4wnh1p-shard-1.temp}]}] on config.transaction_coordinators [2021-10-22T00:32:39.255+0000] [.info] [src/action/restorefrombackup.go:updateShardNameInTransactionCoordinatorsCollection:1873] [00:32:39.255] updating config.transaction_coordinators participants : replacing atlas-4wnh1p-shard-0.temp with atlas-4wnh1p-shard-0 [2021-10-22T00:32:39.256+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.256] Updated all with query [{participants atlas-4wnh1p-shard-0.temp}] and update [{$set [{participants.$ atlas-4wnh1p-shard-0}]}] on config.transaction_coordinators [2021-10-22T00:32:39.256+0000] [.info] [src/action/restorefrombackup.go:updateShardNameInTransactionCoordinatorsCollection:1873] [00:32:39.256] updating config.transaction_coordinators participants : replacing atlas-4wnh1p-shard-1.temp with atlas-4wnh1p-shard-1 [2021-10-22T00:32:39.257+0000] [.info] [src/mongoctl/processctl.go:UpdateAll:3295] [00:32:39.257] Updated all with query [{participants atlas-4wnh1p-shard-1.temp}] and update [{$set [{participants.$ atlas-4wnh1p-shard-1}]}] on config.transaction_coordinators [2021-10-22T00:32:39.257+0000] [.info] [src/action/restorefrombackup.go:clearShardIdentityDocument:1884] [00:32:39.257] Clearing shard identity document [2021-10-22T00:32:39.259+0000] [.info] [src/mongoctl/processctl.go:RemoveAllIfExists:3360] [00:32:39.259] Removed document(s) using query map[_id:shardIdentity] from admin.system.version [2021-10-22T00:32:39.259+0000] [.info] [src/action/restorefrombackup.go:forceUpsertAtmCredsForSeedRestore:2430] [00:32:39.259] Force upserting automation credentials for seed restore [2021-10-22T00:32:39.259+0000] [.info] [src/action/restorefrombackup.go:forceUpsertAtmCreds:2442] [00:32:39.259] Updating automation credentials [2021-10-22T00:32:39.332+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:39.332] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:39.515+0000] [.info] [src/mongoctl/authctl.go:updateUser:349] [00:32:39.515] Updated user AuthUser(mms-automation@admin,roles=AuthRoles(6:{"role":"backup","db":"admin"},{"role":"clusterAdmin","db":"admin"},{"role":"dbAdminAnyDatabase","db":"admin"},{"role":"readWriteAnyDatabase","db":"admin"},{"role":"restore","db":"admin"},{"role":"userAdminAnyDatabase","db":"admin"}),restrictions=[{"clientSource":["127.0.0.1","13.52.26.125","13.56.37.245","18.144.182.252","192.168.248.0/21","204.236.139.94","204.236.170.66","3.101.7.55","50.18.156.198","50.18.90.163","52.8.13.78","52.8.155.179","52.8.49.162","54.151.10.202","54.176.39.164","54.177.210.155","54.193.205.209","54.215.86.105","54.219.98.44","54.241.43.64"]}]) in db admin [2021-10-22T00:32:39.515+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:32:39.515] Initiating stopEphemeralMongo [2021-10-22T00:32:39.515+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:32:39.515] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36897 (local=false) [2021-10-22T00:32:39.515+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:32:39.515] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36897 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:32:39.733+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:32:39.733] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.201743ms. [2021-10-22T00:32:39.733+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:32:39.733] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36897 (local=false) [2021-10-22T00:32:39.733+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:32:39.733] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.858µs. [2021-10-22T00:32:39.733+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:32:39.733] Successfully finished running stopEphemeralMongo [2021-10-22T00:32:39.733+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:32:39.733] Initiating stopEphemeralMongo [2021-10-22T00:32:39.733+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:32:39.733] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:32:40.008+0000] [.info] [src/director/director.go:tracef:794] [00:32:40.008] Running step: 'MarkRestorePhase2Finished' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:32:40.008+0000] [.info] [src/director/director.go:tracef:794] [00:32:40.008] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:32:40.217+0000] [.info] [src/director/director.go:tracef:794] [00:32:40.217] Running step: 'CheckConfigServerNeedsRestorePhase3' of move 'RestoreConfigServerFromBackupPhase3' [2021-10-22T00:32:40.217+0000] [.info] [src/director/director.go:tracef:794] [00:32:40.217] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.IsConfigServer' = true] ['currentState.BackupRestorePhase' = 'Restore Phase 2 Finished'; both are (2)] ] [2021-10-22T00:32:40.217+0000] [.info] [src/director/director.go:tracef:794] [00:32:40.217] Running step: 'WaitCanBounceStop' of move 'RestoreConfigServerFromBackupPhase3' [2021-10-22T00:32:40.217+0000] [.info] [src/director/director.go:tracef:794] [00:32:40.217] because [2021-10-22T00:32:40.218+0000] [.info] [src/director/director.go:tracef:794] [00:32:40.218] Running step: 'BounceStopIfUp' of move 'RestoreConfigServerFromBackupPhase3' [2021-10-22T00:32:40.218+0000] [.info] [src/director/director.go:tracef:794] [00:32:40.218] because ['currentState.CanBounceStop' = true] [2021-10-22T00:32:40.332+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:40.332] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:40.432+0000] [.info] [src/director/director.go:tracef:794] [00:32:40.432] Running step: 'AdjustClusterIdIfNeeded' of move 'RestoreConfigServerFromBackupPhase3' [2021-10-22T00:32:40.432+0000] [.info] [src/director/director.go:tracef:794] [00:32:40.432] because ['currentState.Up' = false] [2021-10-22T00:32:40.653+0000] [.info] [src/director/director.go:tracef:794] [00:32:40.653] Running step: 'WaitCanStartFresh' of move 'RestoreConfigServerFromBackupPhase3' [2021-10-22T00:32:40.653+0000] [.info] [src/director/director.go:tracef:794] [00:32:40.653] because ['currentState.Running' = false] [2021-10-22T00:32:40.653+0000] [.info] [src/director/director.go:tracef:794] [00:32:40.653] Running step: 'StartFresh' of move 'RestoreConfigServerFromBackupPhase3' [2021-10-22T00:32:40.653+0000] [.info] [src/director/director.go:tracef:794] [00:32:40.653] because [All the following are true: ['currentState.Running' = false] ['desiredState.FullVersion' is a member of 'currentState.VersionsOnDisk' ('desiredState.FullVersion'={"trueName":"5.0.3-ent","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","modules":["enterprise"],"major":5,"minor":0,"patch":3}, 'currentState.VersionsOnDisk'=[{"trueName":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","modules":["enterprise"],"major":5,"minor":0,"patch":3}])] ['currentState.IsVCRedistCorrect' = true] ['desiredState.ProcessType' != mongos ('desiredState.ProcessType' = mongod)] ['currentState.StorageFieldsChangeFinished' = true] ['currentState.BackupRestorePhase' = 'Restore Phase 2 Finished'; both are (2)] ['currentState.CanStartFresh' = true] ] [2021-10-22T00:32:40.689+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:32:40.689] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:32:40.690+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:32:40.690] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 27017 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" replication: replSetName: atlas-4wnh1p-config-0 security: authorization: enabled javascriptEnabled: true keyFile: /var/lib/mongodb-mms-automation/keyfile setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "true" watchdogPeriodSeconds: "60" sharding: clusterRole: configsvr storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:32:41.240+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:32:41.240] memory:rss=44531712 alloc=10616912 totalalloc=196750336 mallocs=2545409 frees=2458741 heapinuse=13623296 heapobjects=86668 gcpausetotalns=22354340 gcpauselastns=80805 gcnum=37 utimens=2826929000 stimens=766568000 minflt=22363 majflt=400 [2021-10-22T00:32:41.342+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:32:41.342] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:32:42.129+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:32:42.129] ...Finished start [pid=16381] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":27017,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"replication":{"replSetName":"atlas-4wnh1p-config-0"},"security":{"authorization":"enabled","javascriptEnabled":true,"keyFile":"/var/lib/mongodb-mms-automation/keyfile"},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"true","watchdogPeriodSeconds":"60"},"sharding":{"clusterRole":"configsvr"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:32:42.243+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:32:42.243] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 113.836805ms. [2021-10-22T00:32:42.584+0000] [.info] [src/mongoclientservice/mongoclientservice.go:func1:1573] [00:32:42.584] Testing auth with username __system db=local to atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) connectMode=SingleConnect ipversion=0 tls=true [2021-10-22T00:32:42.703+0000] [.info] [src/mongoctl/processctl.go:GetKeyHashes:2008] [00:32:42.703] Able to successfully auth to atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) using desired auth key [2021-10-22T00:32:43.058+0000] [.info] [src/config/config.go:ReadClusterConfig:509] [00:32:43.058] Backing up cluster config to /var/lib/mongodb-mms-automation/mms-cluster-config-backup.json [2021-10-22T00:32:43.058+0000] [.info] [main/components/agent.go:LoadClusterConfig:262] [00:32:43.058] New cluster config received! 51 ("973422c9462ea20cccc0685177661856--gzip") -> 53 ("3248baa66d6325f50df1304700caf61c--gzip") [2021-10-22T00:32:43.079+0000] [.info] [src/director/director.go:tracef:794] [00:32:43.079] Running step: 'RecordLastRestoreKey' of move 'RestoreConfigServerFromBackupPhase3' [2021-10-22T00:32:43.079+0000] [.info] [src/director/director.go:tracef:794] [00:32:43.079] because ['currentState.Up' = true] [2021-10-22T00:32:43.082+0000] [.info] [src/mongoctl/processctl.go:Update:3279] [00:32:43.082] Updated with query map[] and update [{$set [{lastBackupRestoreUrl }]}] and upsert=true on local.clustermanager [2021-10-22T00:32:43.168+0000] [.info] [src/mongoclientservice/mongoclientservice.go:func1:1573] [00:32:43.168] Testing auth with username __system db=local to atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) connectMode=SingleConnect ipversion=0 tls=true [2021-10-22T00:32:43.220+0000] [.info] [src/mongoctl/processctl.go:GetKeyHashes:2008] [00:32:43.220] Able to successfully auth to atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) using desired auth key [2021-10-22T00:32:43.260+0000] [.info] [state/stateutil/fickleutil.go:func1:3134] [00:32:43.260] Removing move marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started because the move is complete. [2021-10-22T00:32:43.492+0000] [.info] [src/director/director.go:tracef:794] [00:32:43.492] Running step: 'MarkRestoreFinished' of move 'RestoreConfigServerFromBackupPhase3' [2021-10-22T00:32:43.492+0000] [.info] [src/director/director.go:tracef:794] [00:32:43.492] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:32:43.541+0000] [.info] [src/mongoclientservice/mongoclientservice.go:func1:1573] [00:32:43.541] Testing auth with username __system db=local to atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) connectMode=SingleConnect ipversion=0 tls=true [2021-10-22T00:32:43.587+0000] [.info] [src/mongoctl/processctl.go:GetKeyHashes:2008] [00:32:43.587] Able to successfully auth to atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) using desired auth key [2021-10-22T00:32:43.762+0000] [.info] [src/director/director.go:tracef:794] [00:32:43.762] Running step: 'WaitFeatureCompatibilityVersionCorrect' of move 'WaitFeatureCompatibilityVersionCorrect' [2021-10-22T00:32:43.762+0000] [.info] [src/director/director.go:tracef:794] [00:32:43.762] because [All the following are true: ['currentState.Up' = true] ['currentState.isResponsibleForUpdatingFeatureCompatibilityVersion' = false] ] [2021-10-22T00:32:43.763+0000] [.info] [src/director/director.go:tracef:794] [00:32:43.763] Plan executed successfully and finished [2021-10-22T00:32:43.895+0000] [.info] [src/director/director.go:mainLoop:444] [00:32:43.895] Received new cluster config! [2021-10-22T00:32:43.895+0000] [.info] [src/director/director.go:incorporateConfig:302] [00:32:43.895] clusterConfig edition is different ("973422c9462ea20cccc0685177661856--gzip" -> "3248baa66d6325f50df1304700caf61c--gzip"). Incorporating... [2021-10-22T00:32:43.895+0000] [.info] [metrics/collector/realcollector.go:collectLoop:253] [00:32:43.895] Received new cluster config! [2021-10-22T00:32:43.895+0000] [.info] [metrics/collector/realftscollector.go:mainLoop:239] [00:32:43.895] Received new cluster config! [2021-10-22T00:32:43.895+0000] [.info] [metrics/prometheus/exporter.go:mainLoop:73] [00:32:43.895] Received new cluster config! [2021-10-22T00:32:43.895+0000] [.info] [metrics/prometheus/server.go:ApplyNewPromConfig:53] [00:32:43.895] Prometheus config is nil or has prometheus explicitly disabled, stopping the prometheus server if running... [2021-10-22T00:32:43.895+0000] [.info] [metrics/prometheus/server.go:func1:51] [00:32:43.895] ApplyNewPromConfig took 24.854µs [2021-10-22T00:32:43.895+0000] [.info] [realtime/rtcollector/rtcollector.go:controlLoop:95] [00:32:43.895] Received new cluster config! [2021-10-22T00:32:43.895+0000] [.info] [src/dataexplorer/dataexplorer.go:controlLoop:114] [00:32:43.895] Received new cluster config! [2021-10-22T00:32:43.895+0000] [.info] [src/filebeat/filebeat.go:controlLoop:634] [00:32:43.895] Received new cluster config! [2021-10-22T00:32:43.923+0000] [.info] [src/maintainers/externalmaintainer.go:mainLoop:571] [00:32:43.923] Got a new descriptor for mongot [2021-10-22T00:32:43.923+0000] [.info] [src/maintainers/externalmaintainer.go:mainLoop:571] [00:32:43.923] Got a new descriptor for atlasProxy [2021-10-22T00:32:44.034+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:115] [00:32:44.034] Rotating log for mongo process atlas-4wnh1p-config-0-node-1. [2021-10-22T00:32:44.065+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:122] [00:32:44.065] Rotated log for mongo process atlas-4wnh1p-config-0-node-1. [2021-10-22T00:32:44.129+0000] [.info] [src/mongoclientservice/mongoclientservice.go:func1:1573] [00:32:44.129] Testing auth with username __system db=local to atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) connectMode=SingleConnect ipversion=0 tls=true [2021-10-22T00:32:44.191+0000] [.info] [src/mongoctl/processctl.go:GetKeyHashes:2008] [00:32:44.191] Able to successfully auth to atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) using desired auth key [2021-10-22T00:32:44.615+0000] [.info] [src/mongoctl/processctl.go:Update:3279] [00:32:44.615] Updated with query map[] and update [{$set [{agentFeatures [StateCache]} {nextVersion 53}]}] and upsert=true on local.clustermanager [2021-10-22T00:32:44.616+0000] [.info] [src/director/director.go:planAndExecute:537] [00:32:44.616] In Goal State for clusterConfig version: 53 [2021-10-22T00:32:44.617+0000] [.info] [src/mongoctl/processctl.go:Update:3279] [00:32:44.617] Updated with query map[] and update [{$set [{agentFeatures [StateCache]} {lastVersion 53} {nextVersion 53}]}] and upsert=true on local.clustermanager [2021-10-22T00:32:45.004+0000] [.info] [main/components/agent.go:Iterate:852] [00:32:45.004] All 1 Mongo processes are in goal state. Backup in goal state, CPS in goal state [2021-10-22T00:32:45.102+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:32:45.102] clusterConfig unchanged [2021-10-22T00:32:50.173+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:32:50.173] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 17.059562253s [2021-10-22T00:32:54.647+0000] [.info] [src/mongoclientservice/mongoclientservice.go:func1:1573] [00:32:54.647] Testing auth with username __system db=local to atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) connectMode=SingleConnect ipversion=0 tls=true [2021-10-22T00:32:54.685+0000] [.info] [src/mongoctl/processctl.go:GetKeyHashes:2008] [00:32:54.685] Able to successfully auth to atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) using desired auth key [2021-10-22T00:32:55.118+0000] [.info] [main/components/agent.go:Iterate:852] [00:32:55.118] All 1 Mongo processes are in goal state. Backup in goal state, CPS in goal state [2021-10-22T00:32:55.215+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:32:55.215] clusterConfig unchanged [2021-10-22T00:33:01.738+0000] [.info] [src/mongoclientservice/mongoclientservice.go:sendClientToRequesters:729] [00:33:01.738] Retrying connection to atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 (local=false) with new client. old client=0xc000342380 requests=[ 0xc000c9fce0 ] [2021-10-22T00:33:01.944+0000] [.info] [src/mongoctl/processctl.go:Update:3279] [00:33:01.944] Updated with query map[] and update [{$set [{agentFeatures [StateCache]} {nextVersion 53}]}] and upsert=true on local.clustermanager [2021-10-22T00:33:01.960+0000] [.info] [src/director/director.go:computePlan:280] [00:33:01.960] ... process has a plan : WaitHasCorrectAutomationCredentials [2021-10-22T00:33:01.960+0000] [.info] [src/director/director.go:tracef:794] [00:33:01.960] Running step: 'WaitHasCorrectAutomationCredentials' of move 'WaitHasCorrectAutomationCredentials' [2021-10-22T00:33:01.960+0000] [.info] [src/director/director.go:tracef:794] [00:33:01.960] because [All the following are true: [All the following are true: ['currentState.Up' = true] ['currentState.IsAuthSpecified' = true] ['currentState.CanUpdateAutomationCredentials' = false] [All the following are true: [] ] [All the following are true: ['currentState.ReplSetConf.Id' = 'desiredState.ReplSetConf.Id'; both are (atlas-4wnh1p-config-0)] ['currentState.ReplSetConf.Members' = 'desiredState.ReplSetConf.Members'; both are ([{"_id":0,"arbiterOnly":false,"buildIndexes":true,"hidden":false,"horizons":{"MESH":"cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000","PRIVATE":"cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017","PUBLIC":"cluster0-config-00-00.qzt8l.mmscloudteam.com:27017"},"host":"atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017","priority":6.5,"secondaryDelaySecs":0,"tags":{"nodeType":"ELECTABLE","provider":"AWS","region":"US_WEST_1","workloadType":"OPERATIONAL"},"votes":1},{"_id":1,"arbiterOnly":false,"buildIndexes":true,"hidden":false,"horizons":{"MESH":"cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000","PRIVATE":"cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017","PUBLIC":"cluster0-config-00-01.qzt8l.mmscloudteam.com:27017"},"host":"atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017","priority":7,"secondaryDelaySecs":0,"tags":{"nodeType":"ELECTABLE","provider":"AWS","region":"US_WEST_1","workloadType":"OPERATIONAL"},"votes":1},{"_id":2,"arbiterOnly":false,"buildIndexes":true,"hidden":false,"horizons":{"MESH":"cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000","PRIVATE":"cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017","PUBLIC":"cluster0-config-00-02.qzt8l.mmscloudteam.com:27017"},"host":"atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017","priority":7,"secondaryDelaySecs":0,"tags":{"nodeType":"ELECTABLE","provider":"AWS","region":"US_WEST_1","workloadType":"OPERATIONAL"},"votes":1}])] ['currentState.ReplSetConf.ProtocolVersion' = 'desiredState.ReplSetConf.ProtocolVersion'; both are (1)] [] ['desiredState.ReplSetConf.Version' = 0] ] ['currentState.BackupRestoreCorrect' = true] ['current Auth mode' = 'desired Auth mode'; both are (authOn)] ] ['currentState.ProcessHasCorrectAutomationCredentials' = false] ] [2021-10-22T00:33:01.960+0000] [.info] [src/director/director.go:planAndExecute:564] [00:33:01.960] Step=WaitHasCorrectAutomationCredentials as part of Move=WaitHasCorrectAutomationCredentials in plan failed : [00:33:01.960] Postcondition not yet met for step WaitHasCorrectAutomationCredentials because ['currentState.ProcessHasCorrectAutomationCredentials' = false]. Recomputing a plan... [2021-10-22T00:33:02.981+0000] [.info] [src/mongoclientservice/mongoclientservice.go:func1:1573] [00:33:02.981] Testing auth with username __system db=local to atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) connectMode=SingleConnect ipversion=0 tls=true [2021-10-22T00:33:03.017+0000] [.info] [src/mongoctl/processctl.go:GetKeyHashes:2008] [00:33:03.017] Able to successfully auth to atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) using desired auth key [2021-10-22T00:33:03.169+0000] [.info] [src/mongoctl/processctl.go:Update:3279] [00:33:03.169] Updated with query map[] and update [{$set [{agentFeatures [StateCache]} {nextVersion 53}]}] and upsert=true on local.clustermanager [2021-10-22T00:33:03.169+0000] [.info] [src/director/director.go:planAndExecute:537] [00:33:03.169] In Goal State for clusterConfig version: 53 [2021-10-22T00:33:03.170+0000] [.info] [src/mongoctl/processctl.go:Update:3279] [00:33:03.170] Updated with query map[] and update [{$set [{agentFeatures [StateCache]} {lastVersion 53} {nextVersion 53}]}] and upsert=true on local.clustermanager [2021-10-22T00:33:05.327+0000] [.info] [main/components/agent.go:Iterate:852] [00:33:05.326] All 1 Mongo processes are in goal state. Backup in goal state, CPS in goal state [2021-10-22T00:33:05.425+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:33:05.425] clusterConfig unchanged [2021-10-22T00:33:15.468+0000] [.info] [main/components/agent.go:Iterate:852] [00:33:15.468] All 1 Mongo processes are in goal state. Backup in goal state, CPS in goal state [2021-10-22T00:33:15.556+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:33:15.556] clusterConfig unchanged [2021-10-22T00:33:19.362+0000] [.info] [src/mongoclientservice/mongoclientservice.go:sendClientToRequesters:729] [00:33:19.362] Retrying connection to atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) with new client. old client=0xc000efbc00 requests=[ 0xc0002ff800 0xc000db2660 ] [2021-10-22T00:33:19.530+0000] [.info] [src/director/director.go:computePlan:280] [00:33:19.530] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:33:19.530+0000] [.info] [src/director/director.go:tracef:794] [00:33:19.530] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:19.530+0000] [.info] [src/director/director.go:tracef:794] [00:33:19.530] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = -1 is <= phase = 0' = true] ] [2021-10-22T00:33:19.530+0000] [.info] [src/director/director.go:tracef:794] [00:33:19.530] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:19.530+0000] [.info] [src/director/director.go:tracef:794] [00:33:19.530] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:33:19.531+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:33:19.531] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:33:19.688+0000] [.info] [src/director/director.go:tracef:794] [00:33:19.688] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:19.688+0000] [.info] [src/director/director.go:tracef:794] [00:33:19.688] because [2021-10-22T00:33:19.688+0000] [.info] [src/director/director.go:tracef:794] [00:33:19.688] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:19.688+0000] [.info] [src/director/director.go:tracef:794] [00:33:19.688] because ['currentState.CanBounceStop' = true] [2021-10-22T00:33:19.844+0000] [.info] [src/director/director.go:tracef:794] [00:33:19.844] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:19.844+0000] [.info] [src/director/director.go:tracef:794] [00:33:19.844] because [2021-10-22T00:33:19.845+0000] [.info] [src/director/director.go:tracef:794] [00:33:19.845] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:19.845+0000] [.info] [src/director/director.go:tracef:794] [00:33:19.845] because ['currentState.Up' = false] [2021-10-22T00:33:19.845+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:33:19.845] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:33:19.845+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:33:19.845] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:33:19.929+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:33:19.929] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:33:19.937+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:33:19.937] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:33:19.938+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:33:19.938] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:33:19.938+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:33:19.938] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:33:19.940+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:33:19.940] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:33:19.941+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:33:19.941] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:33:19.942+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:33:19.942] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:33:19.945+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:33:19.945] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:33:20.176+0000] [.info] [src/director/director.go:tracef:794] [00:33:20.176] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:20.177+0000] [.info] [src/director/director.go:tracef:794] [00:33:20.177] because ['currentState.Up' = false] [2021-10-22T00:33:20.506+0000] [.info] [src/director/director.go:tracef:794] [00:33:20.506] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:20.507+0000] [.info] [src/director/director.go:tracef:794] [00:33:20.507] because ['currentState.Up' = false] [2021-10-22T00:33:20.507+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:33:20.507] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:33:20.743+0000] [.info] [src/director/director.go:tracef:794] [00:33:20.743] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:20.743+0000] [.info] [src/director/director.go:tracef:794] [00:33:20.743] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:33:20.970+0000] [.info] [src/director/director.go:tracef:794] [00:33:20.970] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:33:20.970+0000] [.info] [src/director/director.go:tracef:794] [00:33:20.970] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:33:20.970+0000] [.info] [src/director/director.go:tracef:794] [00:33:20.970] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:33:20.970+0000] [.info] [src/director/director.go:tracef:794] [00:33:20.970] because ['currentState.Up' = false] [2021-10-22T00:33:20.970+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:33:20.970] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:33:20.970+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:33:20.970] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:33:21.004+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:33:21.004] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:33:21.005+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:33:21.005] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46016 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:33:21.744+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:33:21.744] ...Finished start [pid=29829] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46016,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:33:21.955+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:33:21.955] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 210.811561ms. [2021-10-22T00:33:21.955+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:33:21.955] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46016 (local=false) [2021-10-22T00:33:21.955+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:33:21.955] Dropping existing documents from system.replset [2021-10-22T00:33:22.291+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:33:22.291] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:33:22.318+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:33:22.318] Inserting oplogTruncateAfterPoint document [2021-10-22T00:33:22.318+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:33:22.318] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:33:22.320+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:33:22.320] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:33:22.320+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:33:22.320] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:33:22.393+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:33:22.393] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:33:22.393+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:33:22.393] Setting the minvalid document to [{_id ObjectID("617206d2523370002becc1be")} {t -1} {ts {0 1}}] [2021-10-22T00:33:22.427+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:33:22.427] Inserted [{_id ObjectID("617206d2523370002becc1be")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:33:22.427+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:33:22.427] Inserting new replica set configuration document. [2021-10-22T00:33:22.427+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:33:22.427] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:33:22.442+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:33:22.441] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:33:22.477+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:33:22.477] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:33:22.477+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:33:22.477] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:33:22.477+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:33:22.477] Initiating stopEphemeralMongo [2021-10-22T00:33:22.477+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:33:22.477] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46016 (local=false) [2021-10-22T00:33:22.477+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:33:22.477] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46016 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:33:23.367+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:33:23.367] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.059443ms. [2021-10-22T00:33:23.367+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:33:23.367] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46016 (local=false) [2021-10-22T00:33:23.367+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:33:23.367] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.918µs. [2021-10-22T00:33:23.367+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:33:23.367] Successfully finished running stopEphemeralMongo [2021-10-22T00:33:23.367+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:33:23.367] Successfully stopped existing ephemeral mongod. [2021-10-22T00:33:23.367+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:33:23.367] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:33:23.367+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:33:23.367] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:33:23.401+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:33:23.401] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:33:23.402+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:33:23.402] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41220 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:33:24.672+0000] [.error] [src/action/start.go:func1:93] [103] [00:33:24.672] Error starting mongod : [00:33:24.672] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41220 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:33:23.433Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 30560 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:23.470+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:23.470+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:23.470+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:23.482+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:23.484+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:23.495+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:23.546+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:23.546+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:23.547+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:23.594+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:23.594+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:24.162+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862804:162975][30560:0x7f06b853abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:33:24.257+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862804:257754][30560:0x7f06b853abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:33:24.358+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862804:358528][30560:0x7f06b853abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T00:33:24.478+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862804:478516][30560:0x7f06b853abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:33:24.550+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862804:550673][30560:0x7f06b853abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:33:24.614+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862804:614497][30560:0x7f06b853abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:33:24.614+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862804:614561][30560:0x7f06b853abc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:24.672+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:33:24.672] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:41220 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:33:24.672] Error starting mongod : [00:33:24.672] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41220 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:33:23.433Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 30560 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:23.470+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:23.470+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:23.470+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:23.482+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:23.484+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:23.495+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:23.546+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:23.546+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:23.547+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:23.594+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:23.594+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:24.162+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862804:162975][30560:0x7f06b853abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:33:24.257+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:24.673+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:33:24.673] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:33:24.672] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:41220 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:33:24.672] Error starting mongod : [00:33:24.672] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41220 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:33:23.433Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 30560 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:23.470+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:23.470+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:23.470+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:23.482+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:23.484+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:23.495+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:23.546+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:23.546+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:23.547+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:23.594+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:23.594+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:24.162+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862804:162975][30560:0x7f06b853abc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:24.673+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:33:24.673] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:33:24.673+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:33:24.673] Initiating stopEphemeralMongo [2021-10-22T00:33:24.673+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:33:24.673] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:33:24.673+0000] [.error] [src/director/director.go:executePlan:962] [00:33:24.673] Failed to apply action. Result = : [00:33:24.673] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:33:24.672] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:41220 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:33:24.672] Error starting mongod : [00:33:24.672] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41220 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:33:23.433Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 30560 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:23.470+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:23.470+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:23.470+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:23.482+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:23.484+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:23.495+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:23.546+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:23.546+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:23.547+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:23.594+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:23.594+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:24.162+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:24.674+0000] [.error] [src/director/director.go:planAndExecute:567] [00:33:24.674] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:33:24.673] Failed to apply action. Result = : [00:33:24.673] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:33:24.672] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:41220 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:33:24.672] Error starting mongod : [00:33:24.672] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41220 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:33:23.433Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 30560 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:23.470+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:23.470+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:23.470+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:23.482+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:23.484+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:23.495+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:23.546+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:23.546+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:23.547+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:23.594+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:23.594+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:24.674+0000] [.error] [src/director/director.go:mainLoop:399] [00:33:24.674] Failed to planAndExecute : [00:33:24.674] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:33:24.673] Failed to apply action. Result = : [00:33:24.673] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:33:24.672] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:41220 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:33:24.672] Error starting mongod : [00:33:24.672] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41220 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:33:23.433Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 30560 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:23.470+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:23.470+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:23.470+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:23.482+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:23.484+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:23.495+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:23.546+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:23.546+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:23.547+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:23.594+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:23.594+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:25.804+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:33:25.803] clusterConfig unchanged [2021-10-22T00:33:25.903+0000] [.info] [src/director/director.go:computePlan:280] [00:33:25.903] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:33:25.903+0000] [.info] [src/director/director.go:tracef:794] [00:33:25.903] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:25.903+0000] [.info] [src/director/director.go:tracef:794] [00:33:25.903] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:33:25.903+0000] [.info] [src/director/director.go:tracef:794] [00:33:25.903] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:25.903+0000] [.info] [src/director/director.go:tracef:794] [00:33:25.903] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:33:25.904+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:33:25.904] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:33:26.166+0000] [.info] [src/director/director.go:tracef:794] [00:33:26.166] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:26.167+0000] [.info] [src/director/director.go:tracef:794] [00:33:26.166] because [2021-10-22T00:33:26.167+0000] [.info] [src/director/director.go:tracef:794] [00:33:26.167] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:26.167+0000] [.info] [src/director/director.go:tracef:794] [00:33:26.167] because ['currentState.CanBounceStop' = true] [2021-10-22T00:33:26.364+0000] [.info] [src/mongoclientservice/mongoclientservice.go:sendClientToRequesters:729] [00:33:26.364] Retrying connection to atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) with new client. old client=0xc000a56e00 requests=[ 0xc000c75620 ] [2021-10-22T00:33:26.366+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:26.366] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:26.366+0000] [.info] [metrics/collector/realcollector.go:collectLoop:246] [00:33:26.366] Metrics collection is taking longer than poll sleep interval [2021-10-22T00:33:26.366+0000] [.info] [metrics/collector/util.go:nextPauseTime:22] [00:33:26.366] Metrics collection took 14.008478344s exceeding minimum period 500ms. [2021-10-22T00:33:26.393+0000] [.info] [src/director/director.go:tracef:794] [00:33:26.393] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:26.393+0000] [.info] [src/director/director.go:tracef:794] [00:33:26.393] because [2021-10-22T00:33:26.393+0000] [.info] [src/director/director.go:tracef:794] [00:33:26.393] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:26.393+0000] [.info] [src/director/director.go:tracef:794] [00:33:26.393] because ['currentState.Up' = false] [2021-10-22T00:33:26.393+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:33:26.393] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:33:26.393+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:33:26.393] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:33:26.476+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:33:26.476] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:33:26.478+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:33:26.477] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:33:26.489+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:33:26.489] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:33:26.491+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:33:26.491] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:33:26.492+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:33:26.492] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:33:26.492+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:33:26.492] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:33:26.492+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:33:26.492] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:33:26.495+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:33:26.495] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:33:26.838+0000] [.info] [src/director/director.go:tracef:794] [00:33:26.838] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:26.838+0000] [.info] [src/director/director.go:tracef:794] [00:33:26.838] because ['currentState.Up' = false] [2021-10-22T00:33:26.878+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:26.878] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:27.124+0000] [.info] [src/director/director.go:tracef:794] [00:33:27.124] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:27.125+0000] [.info] [src/director/director.go:tracef:794] [00:33:27.125] because ['currentState.Up' = false] [2021-10-22T00:33:27.125+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:33:27.125] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:33:27.467+0000] [.info] [src/director/director.go:tracef:794] [00:33:27.467] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:27.469+0000] [.info] [src/director/director.go:tracef:794] [00:33:27.469] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:33:27.802+0000] [.info] [src/director/director.go:tracef:794] [00:33:27.802] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:33:27.804+0000] [.info] [src/director/director.go:tracef:794] [00:33:27.804] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:33:27.806+0000] [.info] [src/director/director.go:tracef:794] [00:33:27.806] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:33:27.808+0000] [.info] [src/director/director.go:tracef:794] [00:33:27.808] because ['currentState.Up' = false] [2021-10-22T00:33:27.810+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:33:27.810] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:33:27.810+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:33:27.810] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:33:27.847+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:33:27.847] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:33:27.848+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:33:27.848] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60913 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:33:27.875+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:27.875] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:28.745+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:33:28.745] ...Finished start [pid=33866] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60913,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:33:28.878+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:28.878] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:29.047+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:33:29.047] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 302.036589ms. [2021-10-22T00:33:29.047+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:33:29.047] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:60913 (local=false) [2021-10-22T00:33:29.047+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:33:29.047] Dropping existing documents from system.replset [2021-10-22T00:33:29.375+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:33:29.374] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:33:29.391+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:33:29.391] Inserting oplogTruncateAfterPoint document [2021-10-22T00:33:29.391+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:33:29.391] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:33:29.393+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:33:29.393] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:33:29.393+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:33:29.393] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:33:29.409+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:33:29.409] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:33:29.409+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:33:29.409] Setting the minvalid document to [{_id ObjectID("617206d9523370002becc1cb")} {t -1} {ts {0 1}}] [2021-10-22T00:33:29.422+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:33:29.422] Inserted [{_id ObjectID("617206d9523370002becc1cb")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:33:29.422+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:33:29.422] Inserting new replica set configuration document. [2021-10-22T00:33:29.422+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:33:29.422] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:33:29.452+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:33:29.452] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:33:29.474+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:33:29.474] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:33:29.474+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:33:29.474] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:33:29.475+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:33:29.474] Initiating stopEphemeralMongo [2021-10-22T00:33:29.475+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:33:29.475] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:60913 (local=false) [2021-10-22T00:33:29.475+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:33:29.475] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:60913 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:33:29.878+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:29.878] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:30.012+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:33:30.012] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.142791ms. [2021-10-22T00:33:30.012+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:33:30.012] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:60913 (local=false) [2021-10-22T00:33:30.012+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:33:30.012] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.989µs. [2021-10-22T00:33:30.012+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:33:30.012] Successfully finished running stopEphemeralMongo [2021-10-22T00:33:30.012+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:33:30.012] Successfully stopped existing ephemeral mongod. [2021-10-22T00:33:30.012+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:33:30.012] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:33:30.012+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:33:30.012] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:33:30.067+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:33:30.067] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:33:30.067+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:33:30.067] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46859 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:33:30.909+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:30.909] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:31.409+0000] [.error] [src/action/start.go:func1:93] [103] [00:33:31.409] Error starting mongod : [00:33:31.409] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46859,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46859 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:33:30.108Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 35345 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:30.192+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:30.192+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:30.192+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46859,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:30.198+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:30.200+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:30.210+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:30.277+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:30.277+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:30.278+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:30.325+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:30.325+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:30.815+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862810:815657][35345:0x7fb563d46bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:33:30.924+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862810:924780][35345:0x7fb563d46bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:33:31.027+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862811:27751][35345:0x7fb563d46bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:33:31.207+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862811:207354][35345:0x7fb563d46bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:33:31.279+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862811:279521][35345:0x7fb563d46bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:33:31.336+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862811:336929][35345:0x7fb563d46bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:33:31.337+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862811:337030][35345:0x7fb563d46bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:31.409+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:33:31.409] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46859 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:33:31.409] Error starting mongod : [00:33:31.409] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46859,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46859 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:33:30.108Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 35345 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:30.192+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:30.192+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:30.192+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46859,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:30.198+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:30.200+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:30.210+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:30.277+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:30.277+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:30.278+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:30.325+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:30.325+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:30.815+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862810:815657][35345:0x7fb563d46bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:33:30.924+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:31.410+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:33:31.410] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:33:31.409] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46859 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:33:31.409] Error starting mongod : [00:33:31.409] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46859,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46859 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:33:30.108Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 35345 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:30.192+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:30.192+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:30.192+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46859,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:30.198+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:30.200+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:30.210+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:30.277+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:30.277+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:30.278+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:30.325+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:30.325+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:30.815+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862810:815657][35345:0x7fb563d46bc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:31.410+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:33:31.410] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:33:31.410+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:33:31.410] Initiating stopEphemeralMongo [2021-10-22T00:33:31.410+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:33:31.410] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:33:31.411+0000] [.error] [src/director/director.go:executePlan:962] [00:33:31.410] Failed to apply action. Result = : [00:33:31.410] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:33:31.409] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46859 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:33:31.409] Error starting mongod : [00:33:31.409] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46859,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46859 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:33:30.108Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 35345 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:30.192+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:30.192+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:30.192+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46859,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:30.198+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:30.200+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:30.210+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:30.277+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:30.277+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:30.278+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:30.325+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:30.325+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:30.815+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:31.411+0000] [.error] [src/director/director.go:planAndExecute:567] [00:33:31.411] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:33:31.410] Failed to apply action. Result = : [00:33:31.410] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:33:31.409] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46859 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:33:31.409] Error starting mongod : [00:33:31.409] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46859,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46859 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:33:30.108Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 35345 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:30.192+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:30.192+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:30.192+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46859,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:30.198+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:30.200+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:30.210+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:30.277+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:30.277+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:30.278+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:30.325+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:30.325+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:31.411+0000] [.error] [src/director/director.go:mainLoop:399] [00:33:31.411] Failed to planAndExecute : [00:33:31.411] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:33:31.410] Failed to apply action. Result = : [00:33:31.410] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:33:31.409] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46859 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:33:31.409] Error starting mongod : [00:33:31.409] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46859,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46859 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:33:30.108Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 35345 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:30.192+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:30.192+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:30.192+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46859,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:30.198+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:30.200+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:30.210+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:30.277+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:30.277+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:30.278+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:30.325+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:30.325+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:31.899+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:31.898] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:32.675+0000] [.info] [src/director/director.go:computePlan:280] [00:33:32.675] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:33:32.675+0000] [.info] [src/director/director.go:tracef:794] [00:33:32.675] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:32.675+0000] [.info] [src/director/director.go:tracef:794] [00:33:32.675] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:33:32.676+0000] [.info] [src/director/director.go:tracef:794] [00:33:32.676] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:32.676+0000] [.info] [src/director/director.go:tracef:794] [00:33:32.676] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:33:32.676+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:33:32.676] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:33:32.898+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:32.898] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:33.025+0000] [.info] [src/director/director.go:tracef:794] [00:33:33.025] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:33.025+0000] [.info] [src/director/director.go:tracef:794] [00:33:33.025] because [2021-10-22T00:33:33.025+0000] [.info] [src/director/director.go:tracef:794] [00:33:33.025] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:33.025+0000] [.info] [src/director/director.go:tracef:794] [00:33:33.025] because ['currentState.CanBounceStop' = true] [2021-10-22T00:33:33.383+0000] [.info] [src/director/director.go:tracef:794] [00:33:33.383] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:33.384+0000] [.info] [src/director/director.go:tracef:794] [00:33:33.384] because [2021-10-22T00:33:33.384+0000] [.info] [src/director/director.go:tracef:794] [00:33:33.384] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:33.384+0000] [.info] [src/director/director.go:tracef:794] [00:33:33.384] because ['currentState.Up' = false] [2021-10-22T00:33:33.384+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:33:33.384] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:33:33.384+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:33:33.384] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:33:33.470+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:33:33.470] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:33:33.471+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:33:33.471] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:33:33.484+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:33:33.484] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:33:33.485+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:33:33.485] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:33:33.487+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:33:33.487] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:33:33.487+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:33:33.487] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:33:33.487+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:33:33.487] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:33:33.493+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:33:33.493] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:33:33.876+0000] [.info] [src/director/director.go:tracef:794] [00:33:33.876] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:33.876+0000] [.info] [src/director/director.go:tracef:794] [00:33:33.876] because ['currentState.Up' = false] [2021-10-22T00:33:33.921+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:33.921] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:34.189+0000] [.info] [src/director/director.go:tracef:794] [00:33:34.189] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:34.189+0000] [.info] [src/director/director.go:tracef:794] [00:33:34.189] because ['currentState.Up' = false] [2021-10-22T00:33:34.189+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:33:34.189] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:33:34.487+0000] [.info] [src/director/director.go:tracef:794] [00:33:34.487] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:34.487+0000] [.info] [src/director/director.go:tracef:794] [00:33:34.487] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:33:34.732+0000] [.info] [src/director/director.go:tracef:794] [00:33:34.732] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:33:34.732+0000] [.info] [src/director/director.go:tracef:794] [00:33:34.732] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:33:34.732+0000] [.info] [src/director/director.go:tracef:794] [00:33:34.732] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:33:34.732+0000] [.info] [src/director/director.go:tracef:794] [00:33:34.732] because ['currentState.Up' = false] [2021-10-22T00:33:34.732+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:33:34.732] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:33:34.732+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:33:34.732] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:33:34.774+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:33:34.774] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:33:34.774+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:33:34.774] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49602 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:33:34.952+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:34.952] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:35.925+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:35.925] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:35.997+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:33:35.997] clusterConfig unchanged [2021-10-22T00:33:36.280+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:33:36.279] ...Finished start [pid=39383] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49602,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:33:36.462+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:33:36.462] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 182.721727ms. [2021-10-22T00:33:36.462+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:33:36.462] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49602 (local=false) [2021-10-22T00:33:36.463+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:33:36.462] Dropping existing documents from system.replset [2021-10-22T00:33:36.699+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:33:36.699] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:33:36.719+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:33:36.719] Inserting oplogTruncateAfterPoint document [2021-10-22T00:33:36.719+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:33:36.719] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:33:36.778+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:33:36.778] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:33:36.778+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:33:36.778] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:33:36.868+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:33:36.868] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:33:36.868+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:33:36.868] Setting the minvalid document to [{_id ObjectID("617206e0523370002becc1d8")} {t -1} {ts {0 1}}] [2021-10-22T00:33:36.897+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:33:36.897] Inserted [{_id ObjectID("617206e0523370002becc1d8")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:33:36.897+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:33:36.897] Inserting new replica set configuration document. [2021-10-22T00:33:36.897+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:33:36.897] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:33:36.924+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:36.924] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:36.961+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:33:36.961] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:33:36.988+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:33:36.988] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:33:36.988+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:33:36.988] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:33:36.988+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:33:36.988] Initiating stopEphemeralMongo [2021-10-22T00:33:36.988+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:33:36.988] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49602 (local=false) [2021-10-22T00:33:36.988+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:33:36.988] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49602 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:33:37.143+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:33:37.143] clusterConfig unchanged [2021-10-22T00:33:37.931+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:37.931] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:38.266+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:33:38.266] clusterConfig unchanged [2021-10-22T00:33:38.398+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:33:38.398] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.091125ms. [2021-10-22T00:33:38.398+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:33:38.398] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49602 (local=false) [2021-10-22T00:33:38.399+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:33:38.399] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.879µs. [2021-10-22T00:33:38.399+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:33:38.399] Successfully finished running stopEphemeralMongo [2021-10-22T00:33:38.399+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:33:38.399] Successfully stopped existing ephemeral mongod. [2021-10-22T00:33:38.399+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:33:38.399] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:33:38.399+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:33:38.399] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:33:38.462+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:33:38.462] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:33:38.464+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:33:38.464] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45390 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:33:38.925+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:38.924] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:39.407+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:33:39.407] clusterConfig unchanged [2021-10-22T00:33:39.927+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:39.927] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:40.003+0000] [.error] [src/action/start.go:func1:93] [103] [00:33:40.003] Error starting mongod : [00:33:40.003] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45390,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45390 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 43647 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:38.561+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:38.561+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:38.561+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45390,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:38.564+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:38.565+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:38.576+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:38.644+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:38.644+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:38.648+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:38.697+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:38.697+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:39.411+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862819:411228][43647:0x7f552b55ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:33:39.520+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862819:520130][43647:0x7f552b55ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:33:39.629+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862819:629326][43647:0x7f552b55ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:33:39.746+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862819:746885][43647:0x7f552b55ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:33:39.822+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862819:822929][43647:0x7f552b55ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:33:39.926+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862819:926214][43647:0x7f552b55ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:33:39.926+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862819:926282][43647:0x7f552b55ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:33:39.950+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:40.003+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:33:40.003] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:45390 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:33:40.003] Error starting mongod : [00:33:40.003] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45390,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45390 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 43647 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:38.561+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:38.561+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:38.561+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45390,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:38.564+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:38.565+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:38.576+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:38.644+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:38.644+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:38.648+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:38.697+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:38.697+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:39.411+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862819:411228][43647:0x7f552b55ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:33:39.520+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862819:520130][43647:0x7f552b55ebc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:40.004+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:33:40.004] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:33:40.003] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:45390 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:33:40.003] Error starting mongod : [00:33:40.003] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45390,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45390 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 43647 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:38.561+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:38.561+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:38.561+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45390,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:38.564+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:38.565+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:38.576+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:38.644+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:38.644+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:38.648+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:38.697+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:38.697+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:39.411+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862819:411228][43647:0x7f552b55ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:33:39.520+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:40.004+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:33:40.004] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:33:40.004+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:33:40.004] Initiating stopEphemeralMongo [2021-10-22T00:33:40.004+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:33:40.004] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:33:40.004+0000] [.error] [src/director/director.go:executePlan:962] [00:33:40.004] Failed to apply action. Result = : [00:33:40.004] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:33:40.003] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:45390 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:33:40.003] Error starting mongod : [00:33:40.003] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45390,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45390 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 43647 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:38.561+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:38.561+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:38.561+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45390,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:38.564+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:38.565+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:38.576+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:38.644+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:38.644+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:38.648+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:38.697+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:38.697+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:39.411+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862819:411228][43647:0x7f552b55ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:40.005+0000] [.error] [src/director/director.go:planAndExecute:567] [00:33:40.005] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:33:40.004] Failed to apply action. Result = : [00:33:40.004] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:33:40.003] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:45390 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:33:40.003] Error starting mongod : [00:33:40.003] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45390,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45390 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 43647 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:38.561+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:38.561+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:38.561+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45390,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:38.564+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:38.565+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:38.576+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:38.644+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:38.644+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:38.648+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:38.697+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:38.697+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:39.411+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:40.005+0000] [.error] [src/director/director.go:mainLoop:399] [00:33:40.005] Failed to planAndExecute : [00:33:40.005] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:33:40.004] Failed to apply action. Result = : [00:33:40.004] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:33:40.003] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:45390 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:33:40.003] Error starting mongod : [00:33:40.003] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45390,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45390 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 43647 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:38.561+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:38.561+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:38.561+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45390,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:38.564+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:38.565+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:38.576+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:38.644+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:38.644+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:38.648+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:38.697+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:38.697+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:39.411+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:40.679+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:33:40.679] clusterConfig unchanged [2021-10-22T00:33:40.926+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:40.926] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:41.253+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:33:41.253] memory:rss=49106944 alloc=10008056 totalalloc=462763328 mallocs=6212159 frees=6133673 heapinuse=14770176 heapobjects=78486 gcpausetotalns=35480206 gcpauselastns=1691512 gcnum=79 utimens=8050390000 stimens=2332704000 minflt=32250 majflt=404 [2021-10-22T00:33:41.360+0000] [.info] [src/director/director.go:computePlan:280] [00:33:41.360] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:33:41.361+0000] [.info] [src/director/director.go:tracef:794] [00:33:41.361] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:41.361+0000] [.info] [src/director/director.go:tracef:794] [00:33:41.361] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:33:41.361+0000] [.info] [src/director/director.go:tracef:794] [00:33:41.361] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:41.361+0000] [.info] [src/director/director.go:tracef:794] [00:33:41.361] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:33:41.361+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:33:41.361] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:33:41.731+0000] [.info] [src/director/director.go:tracef:794] [00:33:41.731] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:41.731+0000] [.info] [src/director/director.go:tracef:794] [00:33:41.731] because [2021-10-22T00:33:41.731+0000] [.info] [src/director/director.go:tracef:794] [00:33:41.731] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:41.731+0000] [.info] [src/director/director.go:tracef:794] [00:33:41.731] because ['currentState.CanBounceStop' = true] [2021-10-22T00:33:41.929+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:41.929] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:42.026+0000] [.info] [src/director/director.go:tracef:794] [00:33:42.026] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:42.026+0000] [.info] [src/director/director.go:tracef:794] [00:33:42.026] because [2021-10-22T00:33:42.026+0000] [.info] [src/director/director.go:tracef:794] [00:33:42.026] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:42.026+0000] [.info] [src/director/director.go:tracef:794] [00:33:42.026] because ['currentState.Up' = false] [2021-10-22T00:33:42.026+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:33:42.026] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:33:42.026+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:33:42.026] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:33:42.132+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:33:42.132] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:33:42.145+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:33:42.145] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:33:42.146+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:33:42.146] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:33:42.146+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:33:42.146] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:33:42.148+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:33:42.148] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:33:42.149+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:33:42.149] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:33:42.149+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:33:42.149] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:33:42.151+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:33:42.151] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:33:42.453+0000] [.info] [src/director/director.go:tracef:794] [00:33:42.453] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:42.454+0000] [.info] [src/director/director.go:tracef:794] [00:33:42.454] because ['currentState.Up' = false] [2021-10-22T00:33:42.982+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:42.982] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:43.231+0000] [.info] [src/director/director.go:tracef:794] [00:33:43.231] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:43.231+0000] [.info] [src/director/director.go:tracef:794] [00:33:43.231] because ['currentState.Up' = false] [2021-10-22T00:33:43.231+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:33:43.231] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:33:43.510+0000] [.info] [src/director/director.go:tracef:794] [00:33:43.509] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:43.510+0000] [.info] [src/director/director.go:tracef:794] [00:33:43.510] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:33:43.827+0000] [.info] [src/director/director.go:tracef:794] [00:33:43.827] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:33:43.827+0000] [.info] [src/director/director.go:tracef:794] [00:33:43.827] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:33:43.827+0000] [.info] [src/director/director.go:tracef:794] [00:33:43.827] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:33:43.827+0000] [.info] [src/director/director.go:tracef:794] [00:33:43.827] because ['currentState.Up' = false] [2021-10-22T00:33:43.828+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:33:43.827] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:33:43.828+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:33:43.828] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:33:43.874+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:33:43.874] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:33:43.875+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:33:43.875] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47033 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:33:43.980+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:43.979] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:44.981+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:44.981] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:45.131+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:33:45.131] ...Finished start [pid=49453] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47033,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:33:45.755+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:33:45.755] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 624.338656ms. [2021-10-22T00:33:45.755+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:33:45.755] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47033 (local=false) [2021-10-22T00:33:45.755+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:33:45.755] Dropping existing documents from system.replset [2021-10-22T00:33:45.984+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:45.984] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:47.002+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:47.002] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:47.352+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:33:47.352] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:33:47.404+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:33:47.404] Inserting oplogTruncateAfterPoint document [2021-10-22T00:33:47.404+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:33:47.404] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:33:47.405+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:33:47.405] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:33:47.405+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:33:47.405] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:33:47.434+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:33:47.434] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:33:47.434+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:33:47.434] Setting the minvalid document to [{_id ObjectID("617206eb523370002becc1e6")} {t -1} {ts {0 1}}] [2021-10-22T00:33:47.469+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:33:47.469] Inserted [{_id ObjectID("617206eb523370002becc1e6")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:33:47.469+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:33:47.469] Inserting new replica set configuration document. [2021-10-22T00:33:47.469+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:33:47.469] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:33:47.593+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:33:47.593] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:33:47.797+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:33:47.797] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:33:47.797+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:33:47.797] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:33:47.797+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:33:47.797] Initiating stopEphemeralMongo [2021-10-22T00:33:47.797+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:33:47.797] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47033 (local=false) [2021-10-22T00:33:47.797+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:33:47.797] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47033 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:33:48.003+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:48.003] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:49.003+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:49.003] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:49.614+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:33:49.614] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.001707ms. [2021-10-22T00:33:49.614+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:33:49.614] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47033 (local=false) [2021-10-22T00:33:49.614+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:33:49.614] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 27.094µs. [2021-10-22T00:33:49.614+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:33:49.614] Successfully finished running stopEphemeralMongo [2021-10-22T00:33:49.614+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:33:49.614] Successfully stopped existing ephemeral mongod. [2021-10-22T00:33:49.614+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:33:49.614] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:33:49.614+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:33:49.614] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:33:49.657+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:33:49.657] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:33:49.658+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:33:49.658] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47197 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:33:50.005+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:50.005] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:50.875+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:33:50.875] clusterConfig unchanged [2021-10-22T00:33:51.004+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:51.004] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:51.074+0000] [.error] [src/action/start.go:func1:93] [103] [00:33:51.074] Error starting mongod : [00:33:51.073] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47197,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47197 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 60114 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:49.724+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:49.724+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:49.724+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47197,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:49.726+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:49.728+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:49.733+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:49.764+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:49.765+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:49.766+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:49.825+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:49.825+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:50.464+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862830:464296][60114:0x7f7590389bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:33:50.579+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862830:579122][60114:0x7f7590389bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:33:50.715+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862830:715255][60114:0x7f7590389bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:33:50.845+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862830:845876][60114:0x7f7590389bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:33:50.938+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862830:938275][60114:0x7f7590389bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:33:51.007+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862831:7509][60114:0x7f7590389bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:33:51.007+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862831:7572][60114:0x7f7590389bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:33:51.010+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"in...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:51.074+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:33:51.074] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:47197 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:33:51.074] Error starting mongod : [00:33:51.073] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47197,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47197 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 60114 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:49.724+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:49.724+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:49.724+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47197,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:49.726+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:49.728+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:49.733+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:49.764+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:49.765+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:49.766+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:49.825+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:49.825+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:50.464+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862830:464296][60114:0x7f7590389bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:33:50.579+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862830:579122][60114:0x7f7590389bc0], txn-recover...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:51.074+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:33:51.074] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:33:51.074] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:47197 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:33:51.074] Error starting mongod : [00:33:51.073] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47197,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47197 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 60114 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:49.724+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:49.724+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:49.724+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47197,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:49.726+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:49.728+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:49.733+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:49.764+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:49.765+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:49.766+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:49.825+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:49.825+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:50.464+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862830:464296][60114:0x7f7590389bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:33:50.579+00:00"},"s":"I", "c":"STORAGE", "id":22430, ... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:51.074+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:33:51.074] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:33:51.074+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:33:51.074] Initiating stopEphemeralMongo [2021-10-22T00:33:51.074+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:33:51.074] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:33:51.075+0000] [.error] [src/director/director.go:executePlan:962] [00:33:51.075] Failed to apply action. Result = : [00:33:51.074] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:33:51.074] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:47197 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:33:51.074] Error starting mongod : [00:33:51.073] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47197,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47197 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 60114 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:49.724+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:49.724+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:49.724+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47197,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:49.726+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:49.728+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:49.733+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:49.764+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:49.765+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:49.766+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:49.825+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:49.825+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:50.464+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862830:464296][60114:0x7f7590389bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}}... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:51.075+0000] [.error] [src/director/director.go:planAndExecute:567] [00:33:51.075] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:33:51.075] Failed to apply action. Result = : [00:33:51.074] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:33:51.074] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:47197 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:33:51.074] Error starting mongod : [00:33:51.073] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47197,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47197 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 60114 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:49.724+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:49.724+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:49.724+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47197,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:49.726+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:49.728+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:49.733+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:49.764+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:49.765+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:49.766+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:49.825+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:49.825+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:50.464+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger mes... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:51.076+0000] [.error] [src/director/director.go:mainLoop:399] [00:33:51.076] Failed to planAndExecute : [00:33:51.075] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:33:51.075] Failed to apply action. Result = : [00:33:51.074] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:33:51.074] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:47197 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:33:51.074] Error starting mongod : [00:33:51.073] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47197,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47197 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 60114 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:33:49.724+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:33:49.724+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:33:49.724+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47197,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:33:49.726+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:49.728+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:33:49.733+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:33:49.764+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:49.765+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:49.766+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:33:49.825+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:33:49.825+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:33:50.464+00:00"},"s":"I", ... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:33:52.006+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:52.006] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:52.126+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:33:52.126] clusterConfig unchanged [2021-10-22T00:33:52.508+0000] [.info] [src/director/director.go:computePlan:280] [00:33:52.508] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:33:52.510+0000] [.info] [src/director/director.go:tracef:794] [00:33:52.510] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:52.512+0000] [.info] [src/director/director.go:tracef:794] [00:33:52.512] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:33:52.512+0000] [.info] [src/director/director.go:tracef:794] [00:33:52.512] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:52.512+0000] [.info] [src/director/director.go:tracef:794] [00:33:52.512] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:33:52.512+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:33:52.512] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:33:52.822+0000] [.info] [src/director/director.go:tracef:794] [00:33:52.822] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:52.822+0000] [.info] [src/director/director.go:tracef:794] [00:33:52.822] because [2021-10-22T00:33:52.822+0000] [.info] [src/director/director.go:tracef:794] [00:33:52.822] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:52.822+0000] [.info] [src/director/director.go:tracef:794] [00:33:52.822] because ['currentState.CanBounceStop' = true] [2021-10-22T00:33:53.012+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:53.012] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:53.105+0000] [.info] [src/director/director.go:tracef:794] [00:33:53.105] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:53.105+0000] [.info] [src/director/director.go:tracef:794] [00:33:53.105] because [2021-10-22T00:33:53.105+0000] [.info] [src/director/director.go:tracef:794] [00:33:53.105] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:53.105+0000] [.info] [src/director/director.go:tracef:794] [00:33:53.105] because ['currentState.Up' = false] [2021-10-22T00:33:53.105+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:33:53.105] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:33:53.105+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:33:53.105] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:33:53.189+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:33:53.189] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:33:53.200+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:33:53.200] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:33:53.202+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:33:53.202] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:33:53.203+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:33:53.203] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:33:53.205+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:33:53.204] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:33:53.205+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:33:53.205] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:33:53.205+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:33:53.205] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:33:53.215+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:33:53.215] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:33:53.539+0000] [.info] [src/director/director.go:tracef:794] [00:33:53.539] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:53.539+0000] [.info] [src/director/director.go:tracef:794] [00:33:53.539] because ['currentState.Up' = false] [2021-10-22T00:33:53.821+0000] [.info] [src/director/director.go:tracef:794] [00:33:53.821] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:53.821+0000] [.info] [src/director/director.go:tracef:794] [00:33:53.821] because ['currentState.Up' = false] [2021-10-22T00:33:53.821+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:33:53.821] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:33:54.796+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:54.796] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:55.037+0000] [.info] [src/director/director.go:tracef:794] [00:33:55.037] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:33:55.037+0000] [.info] [src/director/director.go:tracef:794] [00:33:55.037] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:33:55.355+0000] [.info] [src/director/director.go:tracef:794] [00:33:55.354] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:33:55.355+0000] [.info] [src/director/director.go:tracef:794] [00:33:55.355] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:33:55.355+0000] [.info] [src/director/director.go:tracef:794] [00:33:55.355] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:33:55.355+0000] [.info] [src/director/director.go:tracef:794] [00:33:55.355] because ['currentState.Up' = false] [2021-10-22T00:33:55.355+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:33:55.355] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:33:55.355+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:33:55.355] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:33:55.427+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:33:55.427] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:33:55.427+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:33:55.427] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38794 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:33:55.791+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:55.791] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:56.391+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:33:56.391] ...Finished start [pid=67563] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38794,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:33:56.792+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:56.792] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:57.802+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:57.802] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:58.881+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:33:58.881] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 2.490017689s. [2021-10-22T00:33:58.881+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:33:58.881] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38794 (local=false) [2021-10-22T00:33:58.881+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:33:58.881] Dropping existing documents from system.replset [2021-10-22T00:33:58.882+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:58.882] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:33:59.802+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:33:59.802] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:00.689+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:34:00.689] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:34:00.803+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:00.803] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:01.670+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:34:01.670] Inserting oplogTruncateAfterPoint document [2021-10-22T00:34:01.670+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:34:01.670] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:34:01.745+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:34:01.745] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:34:01.745+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:34:01.745] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:34:01.805+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:01.805] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:02.172+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:34:02.172] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:34:02.172+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:34:02.172] Setting the minvalid document to [{_id ObjectID("617206fa523370002becc1f3")} {t -1} {ts {0 1}}] [2021-10-22T00:34:02.212+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:34:02.212] Inserted [{_id ObjectID("617206fa523370002becc1f3")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:34:02.212+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:34:02.212] Inserting new replica set configuration document. [2021-10-22T00:34:02.212+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:34:02.212] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:34:02.230+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:34:02.230] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:34:02.306+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:02.306] clusterConfig unchanged [2021-10-22T00:34:02.806+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:02.806] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:03.272+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:34:03.272] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:34:03.272+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:34:03.272] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:34:03.272+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:34:03.272] Initiating stopEphemeralMongo [2021-10-22T00:34:03.272+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:34:03.272] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38794 (local=false) [2021-10-22T00:34:03.272+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:34:03.272] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38794 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:34:03.410+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:03.410] clusterConfig unchanged [2021-10-22T00:34:03.819+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:03.819] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:04.571+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:04.571] clusterConfig unchanged [2021-10-22T00:34:04.806+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:04.806] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:05.704+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:05.703] clusterConfig unchanged [2021-10-22T00:34:05.808+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:05.808] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:06.808+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:06.808] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:06.815+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:06.815] clusterConfig unchanged [2021-10-22T00:34:07.035+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:34:07.035] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.062717ms. [2021-10-22T00:34:07.035+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:34:07.035] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38794 (local=false) [2021-10-22T00:34:07.036+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:34:07.036] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 29.95µs. [2021-10-22T00:34:07.036+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:34:07.036] Successfully finished running stopEphemeralMongo [2021-10-22T00:34:07.036+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:34:07.036] Successfully stopped existing ephemeral mongod. [2021-10-22T00:34:07.036+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:34:07.036] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:34:07.036+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:34:07.036] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:34:07.086+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:34:07.086] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:34:07.087+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:34:07.087] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59615 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:34:07.810+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:07.810] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:07.948+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:07.948] clusterConfig unchanged [2021-10-22T00:34:08.810+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:08.810] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:09.070+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:09.070] clusterConfig unchanged [2021-10-22T00:34:09.405+0000] [.error] [src/action/start.go:func1:93] [103] [00:34:09.405] Error starting mongod : [00:34:09.405] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59615,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59615 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:07.139Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 17588 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:07.182+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:07.182+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:07.182+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59615,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:07.185+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:07.189+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:07.189+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:07.257+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:07.257+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:07.282+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:07.339+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:07.339+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:08.306+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862848:306956][17588:0x7f7acf4a5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:34:08.438+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862848:438084][17588:0x7f7acf4a5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:34:08.701+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862848:701908][17588:0x7f7acf4a5bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:34:08.955+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862848:955953][17588:0x7f7acf4a5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:34:09.156+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862849:156936][17588:0x7f7acf4a5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:34:09.325+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862849:325212][17588:0x7f7acf4a5bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:34:09.325+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862849:325287][17588:0x7f7acf4a5bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:09.405+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:34:09.405] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59615 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:09.405] Error starting mongod : [00:34:09.405] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59615,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59615 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:07.139Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 17588 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:07.182+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:07.182+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:07.182+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59615,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:07.185+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:07.189+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:07.189+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:07.257+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:07.257+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:07.282+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:07.339+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:07.339+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:08.306+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862848:306956][17588:0x7f7acf4a5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:34:08.438+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:09.405+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:34:09.405] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:34:09.405] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59615 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:09.405] Error starting mongod : [00:34:09.405] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59615,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59615 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:07.139Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 17588 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:07.182+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:07.182+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:07.182+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59615,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:07.185+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:07.189+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:07.189+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:07.257+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:07.257+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:07.282+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:07.339+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:07.339+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:08.306+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862848:306956][17588:0x7f7acf4a5bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:09.405+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:34:09.405] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:34:09.405+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:34:09.405] Initiating stopEphemeralMongo [2021-10-22T00:34:09.405+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:34:09.405] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:34:09.406+0000] [.error] [src/director/director.go:executePlan:962] [00:34:09.406] Failed to apply action. Result = : [00:34:09.405] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:34:09.405] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59615 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:09.405] Error starting mongod : [00:34:09.405] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59615,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59615 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:07.139Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 17588 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:07.182+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:07.182+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:07.182+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59615,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:07.185+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:07.189+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:07.189+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:07.257+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:07.257+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:07.282+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:07.339+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:07.339+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:08.306+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:09.406+0000] [.error] [src/director/director.go:planAndExecute:567] [00:34:09.406] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:34:09.406] Failed to apply action. Result = : [00:34:09.405] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:34:09.405] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59615 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:09.405] Error starting mongod : [00:34:09.405] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59615,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59615 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:07.139Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 17588 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:07.182+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:07.182+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:07.182+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59615,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:07.185+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:07.189+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:07.189+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:07.257+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:07.257+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:07.282+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:07.339+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:07.339+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:09.406+0000] [.error] [src/director/director.go:mainLoop:399] [00:34:09.406] Failed to planAndExecute : [00:34:09.406] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:34:09.406] Failed to apply action. Result = : [00:34:09.405] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:34:09.405] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59615 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:09.405] Error starting mongod : [00:34:09.405] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59615,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59615 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:07.139Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 17588 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:07.182+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:07.182+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:07.182+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59615,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:07.185+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:07.189+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:07.189+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:07.257+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:07.257+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:07.282+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:07.339+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:07.339+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:09.811+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:09.811] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:10.351+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:10.351] clusterConfig unchanged [2021-10-22T00:34:10.839+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:10.839] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:11.127+0000] [.info] [src/director/director.go:computePlan:280] [00:34:11.127] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:34:11.129+0000] [.info] [src/director/director.go:tracef:794] [00:34:11.129] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:11.129+0000] [.info] [src/director/director.go:tracef:794] [00:34:11.129] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:34:11.129+0000] [.info] [src/director/director.go:tracef:794] [00:34:11.129] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:11.129+0000] [.info] [src/director/director.go:tracef:794] [00:34:11.129] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:34:11.129+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:34:11.129] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:34:11.513+0000] [.info] [src/director/director.go:tracef:794] [00:34:11.513] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:11.514+0000] [.info] [src/director/director.go:tracef:794] [00:34:11.514] because [2021-10-22T00:34:11.514+0000] [.info] [src/director/director.go:tracef:794] [00:34:11.514] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:11.514+0000] [.info] [src/director/director.go:tracef:794] [00:34:11.514] because ['currentState.CanBounceStop' = true] [2021-10-22T00:34:12.367+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:12.367] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:12.615+0000] [.info] [src/director/director.go:tracef:794] [00:34:12.615] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:12.615+0000] [.info] [src/director/director.go:tracef:794] [00:34:12.615] because [2021-10-22T00:34:12.615+0000] [.info] [src/director/director.go:tracef:794] [00:34:12.615] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:12.615+0000] [.info] [src/director/director.go:tracef:794] [00:34:12.615] because ['currentState.Up' = false] [2021-10-22T00:34:12.615+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:34:12.615] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:34:12.615+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:34:12.615] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:34:12.704+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:34:12.704] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:34:12.719+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:34:12.719] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:34:12.720+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:34:12.720] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:34:12.722+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:34:12.722] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:34:12.723+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:34:12.723] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:34:12.723+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:34:12.723] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:34:12.723+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:34:12.723] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:34:12.732+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:34:12.732] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:34:15.214+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:34:15.214] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 55.268971412s [2021-10-22T00:34:15.232+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:15.232] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:15.413+0000] [.info] [src/director/director.go:tracef:794] [00:34:15.413] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:15.413+0000] [.info] [src/director/director.go:tracef:794] [00:34:15.413] because ['currentState.Up' = false] [2021-10-22T00:34:15.669+0000] [.info] [src/director/director.go:tracef:794] [00:34:15.669] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:15.670+0000] [.info] [src/director/director.go:tracef:794] [00:34:15.670] because ['currentState.Up' = false] [2021-10-22T00:34:15.670+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:34:15.670] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:34:15.919+0000] [.info] [src/director/director.go:tracef:794] [00:34:15.919] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:15.919+0000] [.info] [src/director/director.go:tracef:794] [00:34:15.919] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:34:16.183+0000] [.info] [src/director/director.go:tracef:794] [00:34:16.183] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:34:16.183+0000] [.info] [src/director/director.go:tracef:794] [00:34:16.183] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:34:16.183+0000] [.info] [src/director/director.go:tracef:794] [00:34:16.183] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:34:16.183+0000] [.info] [src/director/director.go:tracef:794] [00:34:16.183] because ['currentState.Up' = false] [2021-10-22T00:34:16.184+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:34:16.183] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:34:16.184+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:34:16.184] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:34:16.226+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:16.226] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:16.231+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:34:16.231] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:34:16.231+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:34:16.231] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41582 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:34:17.226+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:34:17.226] ...Finished start [pid=31810] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41582,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:34:17.230+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:17.230] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:18.230+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:18.230] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:18.722+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:34:18.722] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 1.495851337s. [2021-10-22T00:34:18.722+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:34:18.722] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41582 (local=false) [2021-10-22T00:34:18.722+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:34:18.722] Dropping existing documents from system.replset [2021-10-22T00:34:19.230+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:19.230] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:20.230+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:20.230] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:20.591+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:20.591] clusterConfig unchanged [2021-10-22T00:34:21.230+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:21.230] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:21.693+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:21.693] clusterConfig unchanged [2021-10-22T00:34:22.231+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:22.230] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:22.825+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:22.825] clusterConfig unchanged [2021-10-22T00:34:22.903+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:34:22.903] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 56.407923371s [2021-10-22T00:34:23.251+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:34:23.251] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:34:23.254+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:23.254] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:23.968+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:23.968] clusterConfig unchanged [2021-10-22T00:34:24.233+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:24.233] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:24.448+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:34:24.448] Inserting oplogTruncateAfterPoint document [2021-10-22T00:34:24.448+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:34:24.448] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:34:24.451+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:34:24.451] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:34:24.451+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:34:24.451] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:34:24.468+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:34:24.468] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:34:24.468+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:34:24.468] Setting the minvalid document to [{_id ObjectID("61720710523370002becc200")} {t -1} {ts {0 1}}] [2021-10-22T00:34:24.483+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:34:24.483] Inserted [{_id ObjectID("61720710523370002becc200")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:34:24.483+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:34:24.483] Inserting new replica set configuration document. [2021-10-22T00:34:24.483+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:34:24.483] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:34:24.496+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:34:24.496] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:34:25.091+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:25.091] clusterConfig unchanged [2021-10-22T00:34:25.233+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:25.233] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:26.223+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:26.223] clusterConfig unchanged [2021-10-22T00:34:26.239+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:26.239] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:26.343+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:34:26.343] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 52.849834841s [2021-10-22T00:34:26.396+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:34:26.396] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:34:26.396+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:34:26.396] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:34:26.396+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:34:26.396] Initiating stopEphemeralMongo [2021-10-22T00:34:26.396+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:34:26.396] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41582 (local=false) [2021-10-22T00:34:26.396+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:34:26.396] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41582 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:34:27.233+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:27.233] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:27.335+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:27.335] clusterConfig unchanged [2021-10-22T00:34:27.821+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:34:27.821] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.142883ms. [2021-10-22T00:34:27.821+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:34:27.821] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41582 (local=false) [2021-10-22T00:34:27.821+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:34:27.821] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 38.894µs. [2021-10-22T00:34:27.821+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:34:27.821] Successfully finished running stopEphemeralMongo [2021-10-22T00:34:27.821+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:34:27.821] Successfully stopped existing ephemeral mongod. [2021-10-22T00:34:27.821+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:34:27.821] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:34:27.821+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:34:27.821] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:34:27.863+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:34:27.863] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:34:27.864+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:34:27.864] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37309 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:34:28.239+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:28.239] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:28.467+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:28.467] clusterConfig unchanged [2021-10-22T00:34:29.136+0000] [.error] [src/action/start.go:func1:93] [103] [00:34:29.136] Error starting mongod : [00:34:29.136] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37309,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37309 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:27.901Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 57574 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:27.924+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:27.924+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:27.924+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37309,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:27.927+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:27.929+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:27.930+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:27.999+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:27.999+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:28.007+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:28.054+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:28.054+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:28.584+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862868:584346][57574:0x7fd5a61c0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:34:28.699+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862868:699797][57574:0x7fd5a61c0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:34:28.814+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862868:814049][57574:0x7fd5a61c0bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:34:28.935+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862868:935590][57574:0x7fd5a61c0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:34:29.013+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862869:13425][57574:0x7fd5a61c0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:34:29.076+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862869:76747][57574:0x7fd5a61c0bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:34:29.076+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862869:76817][57574:0x7fd5a61c0bc0], txn-recover: [WT_VERB_RECOVERY | WT_...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:29.136+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:34:29.136] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37309 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:29.136] Error starting mongod : [00:34:29.136] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37309,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37309 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:27.901Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 57574 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:27.924+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:27.924+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:27.924+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37309,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:27.927+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:27.929+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:27.930+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:27.999+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:27.999+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:28.007+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:28.054+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:28.054+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:28.584+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862868:584346][57574:0x7fd5a61c0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:34:28.699+00:00"},"s":...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:29.137+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:34:29.137] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:34:29.136] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37309 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:29.136] Error starting mongod : [00:34:29.136] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37309,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37309 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:27.901Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 57574 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:27.924+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:27.924+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:27.924+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37309,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:27.927+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:27.929+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:27.930+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:27.999+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:27.999+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:28.007+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:28.054+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:28.054+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:28.584+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862868:584346][57574:0x7fd5a61c0bc0],... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:29.137+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:34:29.137] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:34:29.137+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:34:29.137] Initiating stopEphemeralMongo [2021-10-22T00:34:29.137+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:34:29.137] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:34:29.137+0000] [.error] [src/director/director.go:executePlan:962] [00:34:29.137] Failed to apply action. Result = : [00:34:29.137] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:34:29.136] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37309 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:29.136] Error starting mongod : [00:34:29.136] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37309,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37309 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:27.901Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 57574 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:27.924+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:27.924+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:27.924+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37309,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:27.927+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:27.929+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:27.930+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:27.999+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:27.999+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:28.007+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:28.054+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:28.054+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:28.584+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:29.137+0000] [.error] [src/director/director.go:planAndExecute:567] [00:34:29.137] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:34:29.137] Failed to apply action. Result = : [00:34:29.137] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:34:29.136] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37309 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:29.136] Error starting mongod : [00:34:29.136] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37309,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37309 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:27.901Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 57574 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:27.924+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:27.924+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:27.924+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37309,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:27.927+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:27.929+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:27.930+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:27.999+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:27.999+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:28.007+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:28.054+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:28.054+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:29.137+0000] [.error] [src/director/director.go:mainLoop:399] [00:34:29.137] Failed to planAndExecute : [00:34:29.137] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:34:29.137] Failed to apply action. Result = : [00:34:29.137] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:34:29.136] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37309 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:29.136] Error starting mongod : [00:34:29.136] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37309,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37309 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:27.901Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 57574 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:27.924+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:27.924+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:27.924+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37309,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:27.927+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:27.929+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:27.930+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:27.999+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:27.999+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:28.007+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:28.054+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:28.054+00:00"},"s":"I", "c":"NETWORK", "id":57... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:29.237+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:29.237] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:29.728+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:29.728] clusterConfig unchanged [2021-10-22T00:34:30.242+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:30.242] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:30.412+0000] [.info] [src/director/director.go:computePlan:280] [00:34:30.412] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:34:30.412+0000] [.info] [src/director/director.go:tracef:794] [00:34:30.412] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:30.412+0000] [.info] [src/director/director.go:tracef:794] [00:34:30.412] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:34:30.412+0000] [.info] [src/director/director.go:tracef:794] [00:34:30.412] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:30.412+0000] [.info] [src/director/director.go:tracef:794] [00:34:30.412] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:34:30.412+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:34:30.412] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:34:30.643+0000] [.info] [src/director/director.go:tracef:794] [00:34:30.643] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:30.643+0000] [.info] [src/director/director.go:tracef:794] [00:34:30.643] because [2021-10-22T00:34:30.644+0000] [.info] [src/director/director.go:tracef:794] [00:34:30.644] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:30.644+0000] [.info] [src/director/director.go:tracef:794] [00:34:30.644] because ['currentState.CanBounceStop' = true] [2021-10-22T00:34:30.880+0000] [.info] [src/director/director.go:tracef:794] [00:34:30.880] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:30.880+0000] [.info] [src/director/director.go:tracef:794] [00:34:30.880] because [2021-10-22T00:34:30.881+0000] [.info] [src/director/director.go:tracef:794] [00:34:30.881] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:30.881+0000] [.info] [src/director/director.go:tracef:794] [00:34:30.881] because ['currentState.Up' = false] [2021-10-22T00:34:30.881+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:34:30.881] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:34:30.881+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:34:30.881] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:34:30.969+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:34:30.969] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:34:30.969+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:34:30.969] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:34:30.981+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:34:30.981] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:34:30.983+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:34:30.983] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:34:30.985+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:34:30.985] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:34:30.985+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:34:30.985] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:34:30.985+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:34:30.985] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:34:30.988+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:34:30.988] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:34:31.245+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:31.245] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:31.276+0000] [.info] [src/director/director.go:tracef:794] [00:34:31.276] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:31.276+0000] [.info] [src/director/director.go:tracef:794] [00:34:31.276] because ['currentState.Up' = false] [2021-10-22T00:34:31.518+0000] [.info] [src/director/director.go:tracef:794] [00:34:31.518] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:31.518+0000] [.info] [src/director/director.go:tracef:794] [00:34:31.518] because ['currentState.Up' = false] [2021-10-22T00:34:31.518+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:34:31.518] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:34:31.771+0000] [.info] [src/director/director.go:tracef:794] [00:34:31.770] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:31.771+0000] [.info] [src/director/director.go:tracef:794] [00:34:31.771] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:34:32.025+0000] [.info] [src/director/director.go:tracef:794] [00:34:32.025] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:34:32.025+0000] [.info] [src/director/director.go:tracef:794] [00:34:32.025] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:34:32.025+0000] [.info] [src/director/director.go:tracef:794] [00:34:32.025] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:34:32.025+0000] [.info] [src/director/director.go:tracef:794] [00:34:32.025] because ['currentState.Up' = false] [2021-10-22T00:34:32.025+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:34:32.025] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:34:32.025+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:34:32.025] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:34:32.068+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:34:32.068] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:34:32.069+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:34:32.069] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42838 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:34:32.247+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:32.246] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:32.783+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:34:32.783] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 50.631970712s [2021-10-22T00:34:33.028+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:34:33.028] ...Finished start [pid=64161] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42838,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:34:33.248+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:33.248] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:33.609+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:34:33.609] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 581.171152ms. [2021-10-22T00:34:33.609+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:34:33.609] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42838 (local=false) [2021-10-22T00:34:33.609+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:34:33.609] Dropping existing documents from system.replset [2021-10-22T00:34:34.139+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:34:34.139] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:34:34.142+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:34:34.142] Inserting oplogTruncateAfterPoint document [2021-10-22T00:34:34.142+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:34:34.142] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:34:34.143+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:34:34.143] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:34:34.143+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:34:34.143] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:34:34.159+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:34:34.159] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:34:34.159+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:34:34.159] Setting the minvalid document to [{_id ObjectID("6172071a523370002becc20d")} {t -1} {ts {0 1}}] [2021-10-22T00:34:34.172+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:34:34.172] Inserted [{_id ObjectID("6172071a523370002becc20d")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:34:34.172+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:34:34.172] Inserting new replica set configuration document. [2021-10-22T00:34:34.172+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:34:34.172] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:34:34.186+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:34:34.186] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:34:34.197+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:34:34.197] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:34:34.197+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:34:34.197] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:34:34.197+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:34:34.197] Initiating stopEphemeralMongo [2021-10-22T00:34:34.197+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:34:34.197] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42838 (local=false) [2021-10-22T00:34:34.197+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:34:34.197] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42838 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:34:34.248+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:34.248] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:35.250+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:35.249] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:35.264+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:34:35.264] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.001147ms. [2021-10-22T00:34:35.264+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:34:35.264] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42838 (local=false) [2021-10-22T00:34:35.264+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:34:35.264] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.913µs. [2021-10-22T00:34:35.264+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:34:35.264] Successfully finished running stopEphemeralMongo [2021-10-22T00:34:35.265+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:34:35.265] Successfully stopped existing ephemeral mongod. [2021-10-22T00:34:35.265+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:34:35.265] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:34:35.265+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:34:35.265] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:34:35.308+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:34:35.308] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:34:35.309+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:34:35.308] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36302 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:34:36.251+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:36.251] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:36.573+0000] [.error] [src/action/start.go:func1:93] [103] [00:34:36.572] Error starting mongod : [00:34:36.572] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36302 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:35.345Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 69996 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:35.369+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:35.369+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:35.369+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:35.372+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:35.374+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:35.375+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:35.387+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:35.387+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:35.388+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:35.433+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:35.433+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:36.022+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862876:22066][69996:0x7ff173284bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:34:36.135+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862876:135290][69996:0x7ff173284bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:34:36.246+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862876:246455][69996:0x7ff173284bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T00:34:36.371+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862876:371135][69996:0x7ff173284bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:34:36.450+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862876:450651][69996:0x7ff173284bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:34:36.514+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862876:514556][69996:0x7ff173284bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:34:36.514+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862876:514633][69996:0x7ff173284bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:36.573+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:34:36.573] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36302 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:36.572] Error starting mongod : [00:34:36.572] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36302 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:35.345Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 69996 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:35.369+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:35.369+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:35.369+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:35.372+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:35.374+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:35.375+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:35.387+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:35.387+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:35.388+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:35.433+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:35.433+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:36.022+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862876:22066][69996:0x7ff173284bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:34:36.135+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:36.573+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:34:36.573] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:34:36.573] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36302 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:36.572] Error starting mongod : [00:34:36.572] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36302 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:35.345Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 69996 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:35.369+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:35.369+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:35.369+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:35.372+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:35.374+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:35.375+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:35.387+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:35.387+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:35.388+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:35.433+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:35.433+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:36.022+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862876:22066][69996:0x7ff173284bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:36.573+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:34:36.573] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:34:36.573+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:34:36.573] Initiating stopEphemeralMongo [2021-10-22T00:34:36.573+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:34:36.573] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:34:36.574+0000] [.error] [src/director/director.go:executePlan:962] [00:34:36.574] Failed to apply action. Result = : [00:34:36.573] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:34:36.573] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36302 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:36.572] Error starting mongod : [00:34:36.572] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36302 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:35.345Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 69996 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:35.369+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:35.369+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:35.369+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:35.372+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:35.374+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:35.375+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:35.387+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:35.387+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:35.388+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:35.433+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:35.433+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:36.022+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:36.574+0000] [.error] [src/director/director.go:planAndExecute:567] [00:34:36.574] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:34:36.574] Failed to apply action. Result = : [00:34:36.573] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:34:36.573] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36302 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:36.572] Error starting mongod : [00:34:36.572] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36302 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:35.345Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 69996 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:35.369+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:35.369+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:35.369+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:35.372+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:35.374+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:35.375+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:35.387+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:35.387+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:35.388+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:35.433+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:35.433+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:36.574+0000] [.error] [src/director/director.go:mainLoop:399] [00:34:36.574] Failed to planAndExecute : [00:34:36.574] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:34:36.574] Failed to apply action. Result = : [00:34:36.573] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:34:36.573] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36302 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:36.572] Error starting mongod : [00:34:36.572] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36302 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:35.345Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 69996 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:35.369+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:35.369+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:35.369+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:35.372+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:35.374+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:35.375+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:35.387+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:35.387+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:35.388+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:35.433+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:35.433+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:37.253+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:37.253] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:37.851+0000] [.info] [src/director/director.go:computePlan:280] [00:34:37.851] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:34:37.852+0000] [.info] [src/director/director.go:tracef:794] [00:34:37.852] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:37.852+0000] [.info] [src/director/director.go:tracef:794] [00:34:37.852] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:34:37.852+0000] [.info] [src/director/director.go:tracef:794] [00:34:37.852] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:37.852+0000] [.info] [src/director/director.go:tracef:794] [00:34:37.852] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:34:37.852+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:34:37.852] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:34:38.101+0000] [.info] [src/director/director.go:tracef:794] [00:34:38.101] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:38.101+0000] [.info] [src/director/director.go:tracef:794] [00:34:38.101] because [2021-10-22T00:34:38.101+0000] [.info] [src/director/director.go:tracef:794] [00:34:38.101] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:38.101+0000] [.info] [src/director/director.go:tracef:794] [00:34:38.101] because ['currentState.CanBounceStop' = true] [2021-10-22T00:34:38.263+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:38.263] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:38.367+0000] [.info] [src/director/director.go:tracef:794] [00:34:38.367] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:38.367+0000] [.info] [src/director/director.go:tracef:794] [00:34:38.367] because [2021-10-22T00:34:38.367+0000] [.info] [src/director/director.go:tracef:794] [00:34:38.367] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:38.367+0000] [.info] [src/director/director.go:tracef:794] [00:34:38.367] because ['currentState.Up' = false] [2021-10-22T00:34:38.367+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:34:38.367] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:34:38.367+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:34:38.367] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:34:38.452+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:34:38.452] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:34:38.464+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:34:38.464] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:34:38.465+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:34:38.465] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:34:38.466+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:34:38.466] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:34:38.468+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:34:38.468] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:34:38.468+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:34:38.468] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:34:38.468+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:34:38.468] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:34:38.474+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:34:38.474] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:34:38.736+0000] [.info] [src/director/director.go:tracef:794] [00:34:38.736] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:38.736+0000] [.info] [src/director/director.go:tracef:794] [00:34:38.736] because ['currentState.Up' = false] [2021-10-22T00:34:38.981+0000] [.info] [src/director/director.go:tracef:794] [00:34:38.981] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:38.981+0000] [.info] [src/director/director.go:tracef:794] [00:34:38.981] because ['currentState.Up' = false] [2021-10-22T00:34:38.981+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:34:38.981] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:34:39.249+0000] [.info] [src/director/director.go:tracef:794] [00:34:39.249] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:39.249+0000] [.info] [src/director/director.go:tracef:794] [00:34:39.249] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:34:39.255+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:39.255] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:39.501+0000] [.info] [src/director/director.go:tracef:794] [00:34:39.501] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:34:39.501+0000] [.info] [src/director/director.go:tracef:794] [00:34:39.501] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:34:39.501+0000] [.info] [src/director/director.go:tracef:794] [00:34:39.501] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:34:39.501+0000] [.info] [src/director/director.go:tracef:794] [00:34:39.501] because ['currentState.Up' = false] [2021-10-22T00:34:39.501+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:34:39.501] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:34:39.501+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:34:39.501] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:34:39.543+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:34:39.543] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:34:39.543+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:34:39.543] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50225 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:34:39.940+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:39.940] clusterConfig unchanged [2021-10-22T00:34:40.749+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:34:40.749] ...Finished start [pid=2652] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50225,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:34:40.753+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:40.753] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:40.753+0000] [.info] [metrics/collector/util.go:nextPauseTime:22] [00:34:40.753] Metrics collection took 507.523766ms exceeding minimum period 500ms. [2021-10-22T00:34:40.934+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:34:40.934] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 47.718603806s [2021-10-22T00:34:41.080+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:41.080] clusterConfig unchanged [2021-10-22T00:34:41.186+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:34:41.186] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 436.376845ms. [2021-10-22T00:34:41.186+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:34:41.186] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50225 (local=false) [2021-10-22T00:34:41.186+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:34:41.186] Dropping existing documents from system.replset [2021-10-22T00:34:41.255+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:34:41.255] memory:rss=49881088 alloc=8904040 totalalloc=732750712 mallocs=9807691 frees=9745956 heapinuse=13705216 heapobjects=61735 gcpausetotalns=824894246 gcpauselastns=125564 gcnum=121 utimens=12683226000 stimens=3465253000 minflt=41852 majflt=404 [2021-10-22T00:34:41.264+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:41.264] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:41.555+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:34:41.555] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:34:41.856+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:34:41.855] Inserting oplogTruncateAfterPoint document [2021-10-22T00:34:41.856+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:34:41.856] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:34:41.857+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:34:41.857] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:34:41.857+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:34:41.857] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:34:41.871+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:34:41.871] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:34:41.871+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:34:41.871] Setting the minvalid document to [{_id ObjectID("61720721523370002becc21a")} {t -1} {ts {0 1}}] [2021-10-22T00:34:41.885+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:34:41.885] Inserted [{_id ObjectID("61720721523370002becc21a")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:34:41.885+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:34:41.885] Inserting new replica set configuration document. [2021-10-22T00:34:41.885+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:34:41.885] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:34:41.898+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:34:41.898] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:34:41.908+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:34:41.908] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:34:41.908+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:34:41.908] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:34:41.908+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:34:41.908] Initiating stopEphemeralMongo [2021-10-22T00:34:41.908+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:34:41.908] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50225 (local=false) [2021-10-22T00:34:41.908+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:34:41.908] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50225 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:34:42.252+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:42.252] clusterConfig unchanged [2021-10-22T00:34:42.262+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:42.262] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:43.115+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:34:43.115] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.178358ms. [2021-10-22T00:34:43.115+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:34:43.115] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50225 (local=false) [2021-10-22T00:34:43.115+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:34:43.115] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.818µs. [2021-10-22T00:34:43.115+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:34:43.115] Successfully finished running stopEphemeralMongo [2021-10-22T00:34:43.115+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:34:43.115] Successfully stopped existing ephemeral mongod. [2021-10-22T00:34:43.115+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:34:43.115] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:34:43.115+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:34:43.115] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:34:43.156+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:34:43.156] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:34:43.157+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:34:43.157] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52827 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:34:43.269+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:43.269] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:43.395+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:43.395] clusterConfig unchanged [2021-10-22T00:34:44.271+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:44.271] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:44.424+0000] [.error] [src/action/start.go:func1:93] [103] [00:34:44.424] Error starting mongod : [00:34:44.424] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52827,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52827 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:43.207Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 8949 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:43.233+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:43.233+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:43.233+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52827,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:43.234+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:43.238+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:43.249+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:43.280+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:43.280+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:43.297+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:43.343+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:43.343+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:43.881+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862883:881248][8949:0x7f68cc9cebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:34:43.989+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862883:989506][8949:0x7f68cc9cebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:34:44.100+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862884:100000][8949:0x7f68cc9cebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T00:34:44.229+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862884:229942][8949:0x7f68cc9cebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:34:44.305+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862884:305601][8949:0x7f68cc9cebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:34:44.366+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862884:366400][8949:0x7f68cc9cebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:34:44.366+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862884:366474][8949:0x7f68cc9cebc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:44.424+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:34:44.424] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:52827 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:44.424] Error starting mongod : [00:34:44.424] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52827,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52827 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:43.207Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 8949 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:43.233+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:43.233+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:43.233+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52827,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:43.234+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:43.238+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:43.249+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:43.280+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:43.280+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:43.297+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:43.343+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:43.343+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:43.881+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862883:881248][8949:0x7f68cc9cebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:34:43.989+00:00"},"s...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:44.424+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:34:44.424] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:34:44.424] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:52827 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:44.424] Error starting mongod : [00:34:44.424] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52827,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52827 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:43.207Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 8949 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:43.233+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:43.233+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:43.233+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52827,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:43.234+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:43.238+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:43.249+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:43.280+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:43.280+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:43.297+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:43.343+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:43.343+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:43.881+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862883:881248][8949:0x7f68cc9cebc0... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:44.425+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:34:44.425] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:34:44.425+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:34:44.425] Initiating stopEphemeralMongo [2021-10-22T00:34:44.425+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:34:44.425] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:34:44.425+0000] [.error] [src/director/director.go:executePlan:962] [00:34:44.425] Failed to apply action. Result = : [00:34:44.424] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:34:44.424] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:52827 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:44.424] Error starting mongod : [00:34:44.424] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52827,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52827 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:43.207Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 8949 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:43.233+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:43.233+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:43.233+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52827,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:43.234+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:43.238+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:43.249+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:43.280+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:43.280+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:43.297+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:43.343+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:43.343+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:43.881+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:44.425+0000] [.error] [src/director/director.go:planAndExecute:567] [00:34:44.425] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:34:44.425] Failed to apply action. Result = : [00:34:44.424] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:34:44.424] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:52827 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:44.424] Error starting mongod : [00:34:44.424] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52827,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52827 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:43.207Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 8949 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:43.233+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:43.233+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:43.233+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52827,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:43.234+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:43.238+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:43.249+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:43.280+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:43.280+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:43.297+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:43.343+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:43.343+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:44.425+0000] [.error] [src/director/director.go:mainLoop:399] [00:34:44.425] Failed to planAndExecute : [00:34:44.425] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:34:44.425] Failed to apply action. Result = : [00:34:44.424] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:34:44.424] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:52827 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:44.424] Error starting mongod : [00:34:44.424] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52827,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52827 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:43.207Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 8949 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:43.233+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:43.233+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:43.233+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52827,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:43.234+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:43.238+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:43.249+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:43.280+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:43.280+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:43.297+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:43.343+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:43.343+00:00"},"s":"I", "c":"NETWORK", "id"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:44.675+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:44.675] clusterConfig unchanged [2021-10-22T00:34:45.270+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:45.270] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:45.739+0000] [.info] [src/director/director.go:computePlan:280] [00:34:45.739] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:34:45.739+0000] [.info] [src/director/director.go:tracef:794] [00:34:45.739] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:45.739+0000] [.info] [src/director/director.go:tracef:794] [00:34:45.739] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:34:45.739+0000] [.info] [src/director/director.go:tracef:794] [00:34:45.739] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:45.739+0000] [.info] [src/director/director.go:tracef:794] [00:34:45.739] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:34:45.739+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:34:45.739] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:34:45.979+0000] [.info] [src/director/director.go:tracef:794] [00:34:45.979] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:45.979+0000] [.info] [src/director/director.go:tracef:794] [00:34:45.979] because [2021-10-22T00:34:45.979+0000] [.info] [src/director/director.go:tracef:794] [00:34:45.979] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:45.979+0000] [.info] [src/director/director.go:tracef:794] [00:34:45.979] because ['currentState.CanBounceStop' = true] [2021-10-22T00:34:46.247+0000] [.info] [src/director/director.go:tracef:794] [00:34:46.247] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:46.247+0000] [.info] [src/director/director.go:tracef:794] [00:34:46.247] because [2021-10-22T00:34:46.247+0000] [.info] [src/director/director.go:tracef:794] [00:34:46.247] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:46.247+0000] [.info] [src/director/director.go:tracef:794] [00:34:46.247] because ['currentState.Up' = false] [2021-10-22T00:34:46.247+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:34:46.247] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:34:46.247+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:34:46.247] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:34:46.271+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:46.271] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:46.329+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:34:46.329] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:34:46.341+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:34:46.341] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:34:46.342+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:34:46.342] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:34:46.343+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:34:46.343] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:34:46.345+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:34:46.345] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:34:46.345+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:34:46.345] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:34:46.345+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:34:46.345] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:34:46.349+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:34:46.349] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:34:46.627+0000] [.info] [src/director/director.go:tracef:794] [00:34:46.627] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:46.627+0000] [.info] [src/director/director.go:tracef:794] [00:34:46.627] because ['currentState.Up' = false] [2021-10-22T00:34:46.874+0000] [.info] [src/director/director.go:tracef:794] [00:34:46.874] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:46.874+0000] [.info] [src/director/director.go:tracef:794] [00:34:46.874] because ['currentState.Up' = false] [2021-10-22T00:34:46.874+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:34:46.874] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:34:47.262+0000] [.info] [src/director/director.go:tracef:794] [00:34:47.262] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:47.262+0000] [.info] [src/director/director.go:tracef:794] [00:34:47.262] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:34:48.809+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:48.809] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:48.809+0000] [.info] [metrics/collector/realcollector.go:collectLoop:246] [00:34:48.809] Metrics collection is taking longer than poll sleep interval [2021-10-22T00:34:48.809+0000] [.info] [metrics/collector/util.go:nextPauseTime:22] [00:34:48.809] Metrics collection took 1.545509968s exceeding minimum period 500ms. [2021-10-22T00:34:49.007+0000] [.info] [src/director/director.go:tracef:794] [00:34:49.007] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:34:49.007+0000] [.info] [src/director/director.go:tracef:794] [00:34:49.007] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:34:49.007+0000] [.info] [src/director/director.go:tracef:794] [00:34:49.007] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:34:49.007+0000] [.info] [src/director/director.go:tracef:794] [00:34:49.007] because ['currentState.Up' = false] [2021-10-22T00:34:49.007+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:34:49.007] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:34:49.007+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:34:49.007] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:34:49.054+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:34:49.054] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:34:49.055+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:34:49.055] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33199 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:34:49.326+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:49.326] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:49.911+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:34:49.911] ...Finished start [pid=19455] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33199,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:34:50.332+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:50.332] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:51.327+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:51.327] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:52.329+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:52.329] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:52.370+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:34:52.370] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 2.458717451s. [2021-10-22T00:34:52.370+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:34:52.370] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:33199 (local=false) [2021-10-22T00:34:52.370+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:34:52.370] Dropping existing documents from system.replset [2021-10-22T00:34:53.329+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:53.329] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:53.965+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:34:53.965] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:34:53.971+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:34:53.971] Inserting oplogTruncateAfterPoint document [2021-10-22T00:34:53.971+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:34:53.971] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:34:54.008+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:34:54.008] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:34:54.008+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:34:54.008] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:34:54.329+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:54.329] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:54.577+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:34:54.576] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 41.844713285s [2021-10-22T00:34:54.644+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:34:54.644] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:34:54.644+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:34:54.644] Setting the minvalid document to [{_id ObjectID("6172072e523370002becc228")} {t -1} {ts {0 1}}] [2021-10-22T00:34:54.659+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:34:54.659] Inserted [{_id ObjectID("6172072e523370002becc228")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:34:54.659+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:34:54.659] Inserting new replica set configuration document. [2021-10-22T00:34:54.659+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:34:54.659] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:34:54.675+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:34:54.675] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:34:54.686+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:34:54.686] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:34:54.686+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:34:54.686] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:34:54.686+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:34:54.686] Initiating stopEphemeralMongo [2021-10-22T00:34:54.686+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:34:54.686] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:33199 (local=false) [2021-10-22T00:34:54.686+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:34:54.686] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:33199 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:34:54.944+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:54.944] clusterConfig unchanged [2021-10-22T00:34:55.349+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:55.349] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:55.832+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:34:55.832] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.164903ms. [2021-10-22T00:34:55.832+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:34:55.832] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:33199 (local=false) [2021-10-22T00:34:55.833+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:34:55.833] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.719µs. [2021-10-22T00:34:55.833+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:34:55.833] Successfully finished running stopEphemeralMongo [2021-10-22T00:34:55.833+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:34:55.833] Successfully stopped existing ephemeral mongod. [2021-10-22T00:34:55.833+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:34:55.833] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:34:55.833+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:34:55.833] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:34:55.871+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:34:55.871] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:34:55.872+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:34:55.871] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58727 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:34:56.087+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:56.087] clusterConfig unchanged [2021-10-22T00:34:56.353+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:56.353] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:57.130+0000] [.error] [src/action/start.go:func1:93] [103] [00:34:57.130] Error starting mongod : [00:34:57.130] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58727,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58727 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:55.908Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 33713 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:55.933+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:55.933+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:55.933+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58727,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:55.935+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:55.938+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:55.939+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:55.952+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:55.952+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:55.953+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:55.999+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:55.999+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:56.591+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862896:591160][33713:0x7ff859c9abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:34:56.705+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862896:705181][33713:0x7ff859c9abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:34:56.814+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862896:814270][33713:0x7ff859c9abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:34:56.933+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862896:933701][33713:0x7ff859c9abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:34:57.010+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862897:10517][33713:0x7ff859c9abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:34:57.072+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862897:72269][33713:0x7ff859c9abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:34:57.072+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862897:72342][33713:0x7ff859c9abc0], txn-recover: [WT_VERB_RECOVERY | WT_...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:57.131+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:34:57.131] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:58727 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:57.130] Error starting mongod : [00:34:57.130] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58727,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58727 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:55.908Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 33713 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:55.933+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:55.933+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:55.933+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58727,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:55.935+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:55.938+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:55.939+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:55.952+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:55.952+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:55.953+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:55.999+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:55.999+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:56.591+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862896:591160][33713:0x7ff859c9abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:34:56.705+00:00"},"s":...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:57.132+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:34:57.132] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:34:57.131] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:58727 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:57.130] Error starting mongod : [00:34:57.130] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58727,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58727 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:55.908Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 33713 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:55.933+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:55.933+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:55.933+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58727,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:55.935+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:55.938+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:55.939+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:55.952+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:55.952+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:55.953+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:55.999+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:55.999+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:56.591+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862896:591160][33713:0x7ff859c9abc0],... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:57.132+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:34:57.132] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:34:57.133+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:34:57.132] Initiating stopEphemeralMongo [2021-10-22T00:34:57.133+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:34:57.133] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:34:57.138+0000] [.error] [src/director/director.go:executePlan:962] [00:34:57.138] Failed to apply action. Result = : [00:34:57.132] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:34:57.131] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:58727 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:57.130] Error starting mongod : [00:34:57.130] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58727,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58727 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:55.908Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 33713 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:55.933+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:55.933+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:55.933+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58727,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:55.935+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:55.938+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:55.939+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:55.952+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:55.952+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:55.953+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:55.999+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:55.999+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:56.591+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:57.139+0000] [.error] [src/director/director.go:planAndExecute:567] [00:34:57.139] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:34:57.138] Failed to apply action. Result = : [00:34:57.132] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:34:57.131] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:58727 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:57.130] Error starting mongod : [00:34:57.130] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58727,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58727 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:55.908Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 33713 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:55.933+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:55.933+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:55.933+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58727,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:55.935+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:55.938+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:55.939+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:55.952+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:55.952+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:55.953+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:55.999+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:55.999+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:57.139+0000] [.error] [src/director/director.go:mainLoop:399] [00:34:57.139] Failed to planAndExecute : [00:34:57.139] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:34:57.138] Failed to apply action. Result = : [00:34:57.132] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:34:57.131] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:58727 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:34:57.130] Error starting mongod : [00:34:57.130] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58727,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58727 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:34:55.908Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 33713 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:34:55.933+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:34:55.933+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:34:55.933+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58727,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:34:55.935+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:55.938+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:34:55.939+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:34:55.952+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:55.952+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:34:55.953+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:34:55.999+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:34:55.999+00:00"},"s":"I", "c":"NETWORK", "id":57... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:34:57.219+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:57.219] clusterConfig unchanged [2021-10-22T00:34:57.344+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:57.344] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:58.351+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:58.351] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:58.420+0000] [.info] [src/director/director.go:computePlan:280] [00:34:58.420] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:34:58.421+0000] [.info] [src/director/director.go:tracef:794] [00:34:58.421] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:58.421+0000] [.info] [src/director/director.go:tracef:794] [00:34:58.421] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:34:58.421+0000] [.info] [src/director/director.go:tracef:794] [00:34:58.421] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:58.421+0000] [.info] [src/director/director.go:tracef:794] [00:34:58.421] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:34:58.421+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:34:58.421] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:34:58.469+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:34:58.468] clusterConfig unchanged [2021-10-22T00:34:58.683+0000] [.info] [src/director/director.go:tracef:794] [00:34:58.683] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:58.683+0000] [.info] [src/director/director.go:tracef:794] [00:34:58.683] because [2021-10-22T00:34:58.683+0000] [.info] [src/director/director.go:tracef:794] [00:34:58.683] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:58.683+0000] [.info] [src/director/director.go:tracef:794] [00:34:58.683] because ['currentState.CanBounceStop' = true] [2021-10-22T00:34:58.924+0000] [.info] [src/director/director.go:tracef:794] [00:34:58.924] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:58.924+0000] [.info] [src/director/director.go:tracef:794] [00:34:58.924] because [2021-10-22T00:34:58.925+0000] [.info] [src/director/director.go:tracef:794] [00:34:58.925] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:58.925+0000] [.info] [src/director/director.go:tracef:794] [00:34:58.925] because ['currentState.Up' = false] [2021-10-22T00:34:58.925+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:34:58.925] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:34:58.925+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:34:58.925] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:34:59.011+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:34:59.011] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:34:59.024+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:34:59.024] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:34:59.025+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:34:59.025] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:34:59.025+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:34:59.025] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:34:59.030+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:34:59.030] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:34:59.030+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:34:59.030] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:34:59.038+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:34:59.038] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:34:59.044+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:34:59.044] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:34:59.297+0000] [.info] [src/director/director.go:tracef:794] [00:34:59.297] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:59.297+0000] [.info] [src/director/director.go:tracef:794] [00:34:59.297] because ['currentState.Up' = false] [2021-10-22T00:34:59.352+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:34:59.352] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:34:59.562+0000] [.info] [src/director/director.go:tracef:794] [00:34:59.562] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:59.562+0000] [.info] [src/director/director.go:tracef:794] [00:34:59.562] because ['currentState.Up' = false] [2021-10-22T00:34:59.562+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:34:59.562] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:34:59.821+0000] [.info] [src/director/director.go:tracef:794] [00:34:59.821] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:34:59.821+0000] [.info] [src/director/director.go:tracef:794] [00:34:59.821] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:35:00.078+0000] [.info] [src/director/director.go:tracef:794] [00:35:00.078] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:35:00.078+0000] [.info] [src/director/director.go:tracef:794] [00:35:00.078] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:35:00.079+0000] [.info] [src/director/director.go:tracef:794] [00:35:00.079] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:35:00.079+0000] [.info] [src/director/director.go:tracef:794] [00:35:00.079] because ['currentState.Up' = false] [2021-10-22T00:35:00.079+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:35:00.079] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:35:00.079+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:35:00.079] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:35:00.900+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:00.900] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:00.900+0000] [.info] [metrics/collector/util.go:nextPauseTime:22] [00:35:00.900] Metrics collection took 554.315231ms exceeding minimum period 500ms. [2021-10-22T00:35:00.906+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:35:00.906] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:35:00.909+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:35:00.909] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37907 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:35:01.409+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:01.409] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:01.751+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:35:01.751] ...Finished start [pid=42193] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37907,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:35:02.407+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:02.407] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:03.408+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:03.408] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:03.643+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:35:03.643] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 1.892308557s. [2021-10-22T00:35:03.643+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:35:03.643] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37907 (local=false) [2021-10-22T00:35:03.643+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:35:03.643] Dropping existing documents from system.replset [2021-10-22T00:35:04.409+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:04.408] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:04.712+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:35:04.712] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:35:04.728+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:35:04.728] Inserting oplogTruncateAfterPoint document [2021-10-22T00:35:04.728+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:35:04.728] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:35:05.215+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:35:05.215] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:35:05.215+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:35:05.215] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:35:05.410+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:05.410] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:06.159+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:35:06.159] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:35:06.159+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:35:06.159] Setting the minvalid document to [{_id ObjectID("6172073a523370002becc235")} {t -1} {ts {0 1}}] [2021-10-22T00:35:06.175+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:35:06.175] Inserted [{_id ObjectID("6172073a523370002becc235")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:35:06.175+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:35:06.175] Inserting new replica set configuration document. [2021-10-22T00:35:06.175+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:35:06.175] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:35:06.189+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:35:06.189] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:35:06.211+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:35:06.211] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:35:06.211+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:35:06.211] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:35:06.211+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:35:06.211] Initiating stopEphemeralMongo [2021-10-22T00:35:06.211+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:35:06.211] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37907 (local=false) [2021-10-22T00:35:06.211+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:35:06.211] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37907 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:35:06.411+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:06.411] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:07.412+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:07.412] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:08.416+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:08.416] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:08.660+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:35:08.660] clusterConfig unchanged [2021-10-22T00:35:09.412+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:09.412] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:09.612+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:35:09.612] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.50954ms. [2021-10-22T00:35:09.612+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:35:09.612] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37907 (local=false) [2021-10-22T00:35:09.612+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:35:09.612] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.965µs. [2021-10-22T00:35:09.612+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:35:09.612] Successfully finished running stopEphemeralMongo [2021-10-22T00:35:09.612+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:35:09.612] Successfully stopped existing ephemeral mongod. [2021-10-22T00:35:09.612+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:35:09.612] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:35:09.612+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:35:09.612] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:35:09.657+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:35:09.657] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:35:09.657+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:35:09.657] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60921 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:35:09.795+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:35:09.795] clusterConfig unchanged [2021-10-22T00:35:10.415+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:10.415] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:10.924+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:35:10.924] clusterConfig unchanged [2021-10-22T00:35:11.001+0000] [.error] [src/action/start.go:func1:93] [103] [00:35:11.000] Error starting mongod : [00:35:11.000] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60921,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60921 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 60835 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:09.729+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:09.729+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:09.729+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60921,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:09.732+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:09.735+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:09.735+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:09.807+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:09.807+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:09.809+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:09.855+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:09.855+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:10.418+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862910:418415][60835:0x7f54040a3bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:35:10.541+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862910:541510][60835:0x7f54040a3bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:35:10.657+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862910:657001][60835:0x7f54040a3bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:35:10.785+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862910:785253][60835:0x7f54040a3bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:35:10.867+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862910:867802][60835:0x7f54040a3bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:35:10.937+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862910:937775][60835:0x7f54040a3bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:35:10.937+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862910:937844][60835:0x7f54040a3bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:35:10.941+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:11.001+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:35:11.001] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60921 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:35:11.000] Error starting mongod : [00:35:11.000] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60921,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60921 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 60835 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:09.729+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:09.729+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:09.729+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60921,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:09.732+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:09.735+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:09.735+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:09.807+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:09.807+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:09.809+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:09.855+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:09.855+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:10.418+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862910:418415][60835:0x7f54040a3bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:35:10.541+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862910:541510][60835:0x7f54040a3bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:11.001+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:35:11.001] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:35:11.001] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60921 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:35:11.000] Error starting mongod : [00:35:11.000] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60921,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60921 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 60835 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:09.729+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:09.729+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:09.729+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60921,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:09.732+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:09.735+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:09.735+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:09.807+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:09.807+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:09.809+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:09.855+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:09.855+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:10.418+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862910:418415][60835:0x7f54040a3bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:35:10.541+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:11.001+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:35:11.001] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:35:11.001+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:35:11.001] Initiating stopEphemeralMongo [2021-10-22T00:35:11.001+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:35:11.001] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:35:11.002+0000] [.error] [src/director/director.go:executePlan:962] [00:35:11.002] Failed to apply action. Result = : [00:35:11.001] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:35:11.001] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60921 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:35:11.000] Error starting mongod : [00:35:11.000] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60921,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60921 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 60835 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:09.729+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:09.729+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:09.729+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60921,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:09.732+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:09.735+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:09.735+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:09.807+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:09.807+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:09.809+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:09.855+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:09.855+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:10.418+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862910:418415][60835:0x7f54040a3bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:11.002+0000] [.error] [src/director/director.go:planAndExecute:567] [00:35:11.002] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:35:11.002] Failed to apply action. Result = : [00:35:11.001] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:35:11.001] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60921 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:35:11.000] Error starting mongod : [00:35:11.000] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60921,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60921 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 60835 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:09.729+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:09.729+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:09.729+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60921,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:09.732+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:09.735+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:09.735+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:09.807+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:09.807+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:09.809+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:09.855+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:09.855+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:10.418+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:11.002+0000] [.error] [src/director/director.go:mainLoop:399] [00:35:11.002] Failed to planAndExecute : [00:35:11.002] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:35:11.002] Failed to apply action. Result = : [00:35:11.001] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:35:11.001] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60921 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:35:11.000] Error starting mongod : [00:35:11.000] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60921,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60921 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 60835 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:09.729+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:09.729+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:09.729+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60921,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:09.732+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:09.735+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:09.735+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:09.807+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:09.807+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:09.809+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:09.855+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:09.855+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:10.418+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:11.366+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:35:11.366] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 40.377811396s [2021-10-22T00:35:11.415+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:11.415] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:12.195+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:35:12.194] clusterConfig unchanged [2021-10-22T00:35:12.289+0000] [.info] [src/director/director.go:computePlan:280] [00:35:12.289] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:35:12.289+0000] [.info] [src/director/director.go:tracef:794] [00:35:12.289] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:12.289+0000] [.info] [src/director/director.go:tracef:794] [00:35:12.289] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:35:12.289+0000] [.info] [src/director/director.go:tracef:794] [00:35:12.289] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:12.289+0000] [.info] [src/director/director.go:tracef:794] [00:35:12.289] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:35:12.290+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:35:12.290] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:35:12.423+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:12.423] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:12.518+0000] [.info] [src/director/director.go:tracef:794] [00:35:12.518] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:12.518+0000] [.info] [src/director/director.go:tracef:794] [00:35:12.518] because [2021-10-22T00:35:12.518+0000] [.info] [src/director/director.go:tracef:794] [00:35:12.518] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:12.518+0000] [.info] [src/director/director.go:tracef:794] [00:35:12.518] because ['currentState.CanBounceStop' = true] [2021-10-22T00:35:12.774+0000] [.info] [src/director/director.go:tracef:794] [00:35:12.774] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:12.774+0000] [.info] [src/director/director.go:tracef:794] [00:35:12.774] because [2021-10-22T00:35:12.774+0000] [.info] [src/director/director.go:tracef:794] [00:35:12.774] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:12.774+0000] [.info] [src/director/director.go:tracef:794] [00:35:12.774] because ['currentState.Up' = false] [2021-10-22T00:35:12.775+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:35:12.775] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:35:12.775+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:35:12.775] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:35:12.855+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:35:12.855] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:35:12.867+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:35:12.867] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:35:12.868+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:35:12.868] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:35:12.870+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:35:12.870] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:35:12.871+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:35:12.871] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:35:12.871+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:35:12.871] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:35:12.871+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:35:12.871] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:35:12.876+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:35:12.876] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:35:13.137+0000] [.info] [src/director/director.go:tracef:794] [00:35:13.137] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:13.138+0000] [.info] [src/director/director.go:tracef:794] [00:35:13.138] because ['currentState.Up' = false] [2021-10-22T00:35:13.403+0000] [.info] [src/director/director.go:tracef:794] [00:35:13.403] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:13.403+0000] [.info] [src/director/director.go:tracef:794] [00:35:13.403] because ['currentState.Up' = false] [2021-10-22T00:35:13.403+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:35:13.403] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:35:13.430+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:13.430] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:13.653+0000] [.info] [src/director/director.go:tracef:794] [00:35:13.653] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:13.654+0000] [.info] [src/director/director.go:tracef:794] [00:35:13.654] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:35:14.924+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:14.924] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:15.146+0000] [.info] [src/director/director.go:tracef:794] [00:35:15.146] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:35:15.146+0000] [.info] [src/director/director.go:tracef:794] [00:35:15.146] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:35:15.146+0000] [.info] [src/director/director.go:tracef:794] [00:35:15.146] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:35:15.146+0000] [.info] [src/director/director.go:tracef:794] [00:35:15.146] because ['currentState.Up' = false] [2021-10-22T00:35:15.146+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:35:15.146] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:35:15.146+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:35:15.146] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:35:15.186+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:35:15.186] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:35:15.186+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:35:15.186] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 44655 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:35:15.919+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:15.919] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:16.057+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:35:16.057] ...Finished start [pid=70436] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44655,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:35:16.939+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:16.939] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:16.956+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:35:16.956] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 898.754581ms. [2021-10-22T00:35:16.956+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:35:16.956] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44655 (local=false) [2021-10-22T00:35:16.956+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:35:16.956] Dropping existing documents from system.replset [2021-10-22T00:35:17.920+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:17.920] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:18.450+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:35:18.450] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:35:18.920+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:18.920] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:19.342+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:35:19.341] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 40.867681118s [2021-10-22T00:35:19.647+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:35:19.647] Inserting oplogTruncateAfterPoint document [2021-10-22T00:35:19.647+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:35:19.647] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:35:19.773+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:35:19.773] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:35:19.773+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:35:19.773] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:35:19.921+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:19.921] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:19.936+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:35:19.935] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:35:19.936+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:35:19.936] Setting the minvalid document to [{_id ObjectID("61720747523370002becc242")} {t -1} {ts {0 1}}] [2021-10-22T00:35:19.949+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:35:19.949] Inserted [{_id ObjectID("61720747523370002becc242")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:35:19.949+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:35:19.949] Inserting new replica set configuration document. [2021-10-22T00:35:19.949+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:35:19.949] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:35:19.964+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:35:19.963] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:35:19.985+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:35:19.985] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:35:19.985+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:35:19.985] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:35:19.985+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:35:19.985] Initiating stopEphemeralMongo [2021-10-22T00:35:19.985+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:35:19.985] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44655 (local=false) [2021-10-22T00:35:19.985+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:35:19.985] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44655 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:35:20.921+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:20.921] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:21.770+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:35:21.770] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.246366ms. [2021-10-22T00:35:21.770+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:35:21.770] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44655 (local=false) [2021-10-22T00:35:21.770+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:35:21.770] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.839µs. [2021-10-22T00:35:21.771+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:35:21.771] Successfully finished running stopEphemeralMongo [2021-10-22T00:35:21.771+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:35:21.771] Successfully stopped existing ephemeral mongod. [2021-10-22T00:35:21.771+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:35:21.771] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:35:21.771+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:35:21.771] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:35:21.813+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:35:21.813] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:35:21.814+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:35:21.814] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49867 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:35:21.942+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:21.942] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:22.395+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:35:22.395] clusterConfig unchanged [2021-10-22T00:35:22.939+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:22.939] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:23.078+0000] [.error] [src/action/start.go:func1:93] [103] [00:35:23.078] Error starting mongod : [00:35:23.078] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49867,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49867 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 10134 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:21.876+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:21.876+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:21.876+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49867,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:21.879+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:21.881+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:21.882+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:21.893+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:21.893+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:21.910+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:21.955+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:21.955+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:22.540+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862922:540808][10134:0x7fa554564bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:35:22.651+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862922:651252][10134:0x7fa554564bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:35:22.759+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862922:759107][10134:0x7fa554564bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:35:22.883+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862922:883557][10134:0x7fa554564bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:35:22.959+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862922:959394][10134:0x7fa554564bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:35:23.022+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862923:22558][10134:0x7fa554564bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:35:23.022+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862923:22625][10134:0x7fa554564bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:35:23.025+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"in...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:23.079+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:35:23.079] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49867 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:35:23.078] Error starting mongod : [00:35:23.078] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49867,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49867 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 10134 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:21.876+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:21.876+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:21.876+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49867,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:21.879+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:21.881+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:21.882+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:21.893+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:21.893+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:21.910+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:21.955+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:21.955+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:22.540+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862922:540808][10134:0x7fa554564bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:35:22.651+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862922:651252][10134:0x7fa554564bc0], txn-recover: ...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:23.079+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:35:23.079] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:35:23.079] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49867 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:35:23.078] Error starting mongod : [00:35:23.078] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49867,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49867 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 10134 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:21.876+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:21.876+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:21.876+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49867,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:21.879+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:21.881+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:21.882+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:21.893+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:21.893+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:21.910+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:21.955+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:21.955+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:22.540+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862922:540808][10134:0x7fa554564bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:35:22.651+00:00"},"s":"I", "c":"STORAGE", "id":22430, "... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:23.079+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:35:23.079] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:35:23.079+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:35:23.079] Initiating stopEphemeralMongo [2021-10-22T00:35:23.079+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:35:23.079] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:35:23.080+0000] [.error] [src/director/director.go:executePlan:962] [00:35:23.080] Failed to apply action. Result = : [00:35:23.079] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:35:23.079] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49867 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:35:23.078] Error starting mongod : [00:35:23.078] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49867,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49867 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 10134 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:21.876+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:21.876+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:21.876+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49867,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:21.879+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:21.881+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:21.882+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:21.893+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:21.893+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:21.910+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:21.955+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:21.955+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:22.540+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862922:540808][10134:0x7fa554564bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:23.080+0000] [.error] [src/director/director.go:planAndExecute:567] [00:35:23.080] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:35:23.080] Failed to apply action. Result = : [00:35:23.079] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:35:23.079] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49867 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:35:23.078] Error starting mongod : [00:35:23.078] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49867,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49867 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 10134 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:21.876+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:21.876+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:21.876+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49867,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:21.879+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:21.881+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:21.882+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:21.893+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:21.893+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:21.910+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:21.955+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:21.955+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:22.540+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger messa... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:23.080+0000] [.error] [src/director/director.go:mainLoop:399] [00:35:23.080] Failed to planAndExecute : [00:35:23.080] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:35:23.080] Failed to apply action. Result = : [00:35:23.079] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:35:23.079] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49867 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:35:23.078] Error starting mongod : [00:35:23.078] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49867,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49867 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 10134 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:21.876+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:21.876+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:21.876+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49867,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:21.879+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:21.881+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:21.882+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:21.893+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:21.893+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:21.910+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:21.955+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:21.955+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:22.540+00:00"},"s":"I", "c... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:23.667+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:35:23.667] clusterConfig unchanged [2021-10-22T00:35:23.932+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:23.932] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:24.723+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:35:24.723] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 38.37381504s [2021-10-22T00:35:24.932+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:24.932] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:25.932+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:25.932] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:26.934+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:26.934] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:27.935+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:27.935] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:28.950+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:28.950] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:29.315+0000] [.info] [src/director/director.go:computePlan:280] [00:35:29.315] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:35:29.315+0000] [.info] [src/director/director.go:tracef:794] [00:35:29.315] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:29.315+0000] [.info] [src/director/director.go:tracef:794] [00:35:29.315] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:35:29.315+0000] [.info] [src/director/director.go:tracef:794] [00:35:29.315] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:29.315+0000] [.info] [src/director/director.go:tracef:794] [00:35:29.315] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:35:29.315+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:35:29.315] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:35:29.533+0000] [.info] [src/director/director.go:tracef:794] [00:35:29.533] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:29.534+0000] [.info] [src/director/director.go:tracef:794] [00:35:29.534] because [2021-10-22T00:35:29.534+0000] [.info] [src/director/director.go:tracef:794] [00:35:29.534] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:29.534+0000] [.info] [src/director/director.go:tracef:794] [00:35:29.534] because ['currentState.CanBounceStop' = true] [2021-10-22T00:35:29.769+0000] [.info] [src/director/director.go:tracef:794] [00:35:29.769] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:29.769+0000] [.info] [src/director/director.go:tracef:794] [00:35:29.769] because [2021-10-22T00:35:29.769+0000] [.info] [src/director/director.go:tracef:794] [00:35:29.769] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:29.769+0000] [.info] [src/director/director.go:tracef:794] [00:35:29.769] because ['currentState.Up' = false] [2021-10-22T00:35:29.769+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:35:29.769] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:35:29.769+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:35:29.769] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:35:29.859+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:35:29.859] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:35:29.871+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:35:29.871] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:35:29.872+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:35:29.872] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:35:29.873+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:35:29.873] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:35:29.875+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:35:29.875] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:35:29.875+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:35:29.875] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:35:29.875+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:35:29.875] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:35:29.880+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:35:29.880] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:35:29.965+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:29.965] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:30.320+0000] [.info] [src/director/director.go:tracef:794] [00:35:30.320] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:30.322+0000] [.info] [src/director/director.go:tracef:794] [00:35:30.322] because ['currentState.Up' = false] [2021-10-22T00:35:30.667+0000] [.info] [src/director/director.go:tracef:794] [00:35:30.667] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:30.669+0000] [.info] [src/director/director.go:tracef:794] [00:35:30.669] because ['currentState.Up' = false] [2021-10-22T00:35:30.670+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:35:30.670] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:35:30.944+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:30.944] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:30.983+0000] [.info] [src/director/director.go:tracef:794] [00:35:30.983] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:30.985+0000] [.info] [src/director/director.go:tracef:794] [00:35:30.985] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:35:31.366+0000] [.info] [src/director/director.go:tracef:794] [00:35:31.366] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:35:31.366+0000] [.info] [src/director/director.go:tracef:794] [00:35:31.366] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:35:31.366+0000] [.info] [src/director/director.go:tracef:794] [00:35:31.366] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:35:31.366+0000] [.info] [src/director/director.go:tracef:794] [00:35:31.366] because ['currentState.Up' = false] [2021-10-22T00:35:31.366+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:35:31.366] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:35:31.366+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:35:31.366] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:35:31.457+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:35:31.457] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:35:31.458+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:35:31.457] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34014 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:35:31.952+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:31.952] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:32.347+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:35:32.347] ...Finished start [pid=28435] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34014,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:35:32.474+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:35:32.474] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 33.429620634s [2021-10-22T00:35:32.528+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:35:32.528] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 181.394733ms. [2021-10-22T00:35:32.528+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:35:32.528] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34014 (local=false) [2021-10-22T00:35:32.528+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:35:32.528] Dropping existing documents from system.replset [2021-10-22T00:35:32.765+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:35:32.765] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:35:32.803+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:35:32.803] Inserting oplogTruncateAfterPoint document [2021-10-22T00:35:32.803+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:35:32.803] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:35:32.804+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:35:32.804] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:35:32.804+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:35:32.804] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:35:32.818+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:35:32.818] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:35:32.818+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:35:32.818] Setting the minvalid document to [{_id ObjectID("61720754523370002becc24f")} {t -1} {ts {0 1}}] [2021-10-22T00:35:32.832+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:35:32.832] Inserted [{_id ObjectID("61720754523370002becc24f")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:35:32.833+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:35:32.833] Inserting new replica set configuration document. [2021-10-22T00:35:32.833+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:35:32.833] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:35:32.848+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:35:32.848] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:35:32.873+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:35:32.873] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:35:32.873+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:35:32.873] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:35:32.873+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:35:32.873] Initiating stopEphemeralMongo [2021-10-22T00:35:32.873+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:35:32.873] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34014 (local=false) [2021-10-22T00:35:32.873+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:35:32.873] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34014 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:35:32.955+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:32.955] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:33.433+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:35:33.433] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.198181ms. [2021-10-22T00:35:33.434+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:35:33.433] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34014 (local=false) [2021-10-22T00:35:33.434+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:35:33.434] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 38.697µs. [2021-10-22T00:35:33.434+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:35:33.434] Successfully finished running stopEphemeralMongo [2021-10-22T00:35:33.434+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:35:33.434] Successfully stopped existing ephemeral mongod. [2021-10-22T00:35:33.434+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:35:33.434] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:35:33.434+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:35:33.434] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:35:33.472+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:35:33.472] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:35:33.473+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:35:33.473] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37730 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:35:33.955+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:35:33.955] clusterConfig unchanged [2021-10-22T00:35:33.955+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:33.955] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:34.701+0000] [.error] [src/action/start.go:func1:93] [103] [00:35:34.701] Error starting mongod : [00:35:34.701] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37730,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37730 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:35:33.509Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 30886 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:33.532+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:33.532+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:33.532+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37730,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:33.535+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:33.537+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:33.538+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:33.555+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:33.555+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:33.557+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:33.602+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:33.602+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:34.165+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862934:165471][30886:0x7f2fd5ec9bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:35:34.274+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862934:274351][30886:0x7f2fd5ec9bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:35:34.385+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862934:385055][30886:0x7f2fd5ec9bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/33920 to 2/256"}} {"t":{"$date":"2021-10-22T00:35:34.501+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862934:501738][30886:0x7f2fd5ec9bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:35:34.577+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862934:576920][30886:0x7f2fd5ec9bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:35:34.647+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862934:647250][30886:0x7f2fd5ec9bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:35:34.647+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862934:647326][30886:0x7f2fd5ec9bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:34.702+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:35:34.702] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37730 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:35:34.701] Error starting mongod : [00:35:34.701] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37730,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37730 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:35:33.509Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 30886 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:33.532+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:33.532+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:33.532+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37730,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:33.535+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:33.537+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:33.538+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:33.555+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:33.555+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:33.557+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:33.602+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:33.602+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:34.165+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862934:165471][30886:0x7f2fd5ec9bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:35:34.274+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:34.702+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:35:34.702] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:35:34.702] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37730 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:35:34.701] Error starting mongod : [00:35:34.701] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37730,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37730 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:35:33.509Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 30886 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:33.532+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:33.532+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:33.532+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37730,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:33.535+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:33.537+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:33.538+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:33.555+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:33.555+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:33.557+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:33.602+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:33.602+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:34.165+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862934:165471][30886:0x7f2fd5ec9bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:34.702+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:35:34.702] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:35:34.702+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:35:34.702] Initiating stopEphemeralMongo [2021-10-22T00:35:34.702+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:35:34.702] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:35:34.703+0000] [.error] [src/director/director.go:executePlan:962] [00:35:34.703] Failed to apply action. Result = : [00:35:34.702] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:35:34.702] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37730 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:35:34.701] Error starting mongod : [00:35:34.701] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37730,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37730 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:35:33.509Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 30886 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:33.532+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:33.532+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:33.532+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37730,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:33.535+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:33.537+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:33.538+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:33.555+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:33.555+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:33.557+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:33.602+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:33.602+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:34.165+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:34.703+0000] [.error] [src/director/director.go:planAndExecute:567] [00:35:34.703] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:35:34.703] Failed to apply action. Result = : [00:35:34.702] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:35:34.702] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37730 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:35:34.701] Error starting mongod : [00:35:34.701] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37730,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37730 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:35:33.509Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 30886 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:33.532+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:33.532+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:33.532+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37730,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:33.535+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:33.537+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:33.538+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:33.555+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:33.555+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:33.557+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:33.602+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:33.602+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:34.703+0000] [.error] [src/director/director.go:mainLoop:399] [00:35:34.703] Failed to planAndExecute : [00:35:34.703] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:35:34.703] Failed to apply action. Result = : [00:35:34.702] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:35:34.702] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37730 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:35:34.701] Error starting mongod : [00:35:34.701] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37730,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37730 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:35:33.509Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 30886 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:33.532+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:33.532+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:33.532+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37730,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:33.535+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:33.537+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:33.538+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:33.555+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:33.555+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:33.557+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:33.602+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:33.602+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:34.955+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:34.955] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:35.216+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:35:35.216] clusterConfig unchanged [2021-10-22T00:35:35.955+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:35.955] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:36.957+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:36.957] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:37.958+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:37.958] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:38.959+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:38.959] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:39.867+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:35:39.867] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 26.990886985s [2021-10-22T00:35:39.960+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:39.960] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:40.911+0000] [.info] [src/director/director.go:computePlan:280] [00:35:40.910] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:35:40.911+0000] [.info] [src/director/director.go:tracef:794] [00:35:40.911] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:40.911+0000] [.info] [src/director/director.go:tracef:794] [00:35:40.911] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:35:40.911+0000] [.info] [src/director/director.go:tracef:794] [00:35:40.911] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:40.911+0000] [.info] [src/director/director.go:tracef:794] [00:35:40.911] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:35:40.911+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:35:40.911] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:35:40.961+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:40.961] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:41.137+0000] [.info] [src/director/director.go:tracef:794] [00:35:41.137] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:41.137+0000] [.info] [src/director/director.go:tracef:794] [00:35:41.137] because [2021-10-22T00:35:41.137+0000] [.info] [src/director/director.go:tracef:794] [00:35:41.137] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:41.138+0000] [.info] [src/director/director.go:tracef:794] [00:35:41.137] because ['currentState.CanBounceStop' = true] [2021-10-22T00:35:41.256+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:35:41.256] memory:rss=50102272 alloc=6572560 totalalloc=996738456 mallocs=13284485 frees=13244274 heapinuse=13516800 heapobjects=40211 gcpausetotalns=1368083361 gcpauselastns=94600 gcnum=161 utimens=17309606000 stimens=4673313000 minflt=51685 majflt=404 [2021-10-22T00:35:41.347+0000] [.info] [src/director/director.go:tracef:794] [00:35:41.347] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:41.348+0000] [.info] [src/director/director.go:tracef:794] [00:35:41.348] because [2021-10-22T00:35:41.348+0000] [.info] [src/director/director.go:tracef:794] [00:35:41.348] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:41.348+0000] [.info] [src/director/director.go:tracef:794] [00:35:41.348] because ['currentState.Up' = false] [2021-10-22T00:35:41.348+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:35:41.348] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:35:41.348+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:35:41.348] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:35:41.428+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:35:41.428] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:35:41.428+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:35:41.428] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:35:41.440+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:35:41.440] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:35:41.442+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:35:41.442] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:35:41.444+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:35:41.444] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:35:41.444+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:35:41.444] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:35:41.444+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:35:41.444] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:35:41.445+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:35:41.445] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:35:41.698+0000] [.info] [src/director/director.go:tracef:794] [00:35:41.698] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:41.699+0000] [.info] [src/director/director.go:tracef:794] [00:35:41.699] because ['currentState.Up' = false] [2021-10-22T00:35:41.922+0000] [.info] [src/director/director.go:tracef:794] [00:35:41.922] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:41.922+0000] [.info] [src/director/director.go:tracef:794] [00:35:41.922] because ['currentState.Up' = false] [2021-10-22T00:35:41.922+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:35:41.922] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:35:41.974+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:41.974] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:42.174+0000] [.info] [src/director/director.go:tracef:794] [00:35:42.174] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:42.174+0000] [.info] [src/director/director.go:tracef:794] [00:35:42.174] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:35:42.404+0000] [.info] [src/director/director.go:tracef:794] [00:35:42.404] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:35:42.405+0000] [.info] [src/director/director.go:tracef:794] [00:35:42.405] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:35:42.405+0000] [.info] [src/director/director.go:tracef:794] [00:35:42.405] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:35:42.405+0000] [.info] [src/director/director.go:tracef:794] [00:35:42.405] because ['currentState.Up' = false] [2021-10-22T00:35:42.405+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:35:42.405] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:35:42.405+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:35:42.405] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:35:42.446+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:35:42.446] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:35:42.447+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:35:42.447] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55648 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:35:42.962+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:42.962] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:43.215+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:35:43.215] ...Finished start [pid=48271] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55648,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:35:43.358+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:35:43.358] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 142.50461ms. [2021-10-22T00:35:43.358+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:35:43.358] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:55648 (local=false) [2021-10-22T00:35:43.358+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:35:43.358] Dropping existing documents from system.replset [2021-10-22T00:35:43.497+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:35:43.497] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:35:43.503+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:35:43.503] Inserting oplogTruncateAfterPoint document [2021-10-22T00:35:43.503+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:35:43.503] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:35:43.505+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:35:43.505] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:35:43.505+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:35:43.505] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:35:43.519+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:35:43.519] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:35:43.519+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:35:43.519] Setting the minvalid document to [{_id ObjectID("6172075f523370002becc25c")} {t -1} {ts {0 1}}] [2021-10-22T00:35:43.532+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:35:43.532] Inserted [{_id ObjectID("6172075f523370002becc25c")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:35:43.533+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:35:43.532] Inserting new replica set configuration document. [2021-10-22T00:35:43.533+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:35:43.533] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:35:43.545+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:35:43.545] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:35:43.551+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:35:43.551] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:35:43.551+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:35:43.551] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:35:43.551+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:35:43.551] Initiating stopEphemeralMongo [2021-10-22T00:35:43.551+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:35:43.551] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:55648 (local=false) [2021-10-22T00:35:43.551+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:35:43.551] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:55648 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:35:43.960+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:35:43.960] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.086616ms. [2021-10-22T00:35:43.960+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:35:43.960] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:55648 (local=false) [2021-10-22T00:35:43.960+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:35:43.960] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.387µs. [2021-10-22T00:35:43.960+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:35:43.960] Successfully finished running stopEphemeralMongo [2021-10-22T00:35:43.960+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:35:43.960] Successfully stopped existing ephemeral mongod. [2021-10-22T00:35:43.960+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:35:43.960] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:35:43.960+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:35:43.960] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:35:43.970+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:43.970] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:44.000+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:35:44.000] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:35:44.001+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:35:44.001] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56063 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:35:44.078+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [00:35:44.078] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:56063 (local=false) because it is not up [2021-10-22T00:35:44.974+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:44.974] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:45.235+0000] [.error] [src/action/start.go:func1:93] [103] [00:35:45.235] Error starting mongod : [00:35:45.235] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56063,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56063 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:35:44.036Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 50166 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:44.061+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:44.061+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:44.061+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56063,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:44.063+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:44.065+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:44.067+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:44.113+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:44.113+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:44.114+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:44.160+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:44.160+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:44.696+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862944:696767][50166:0x7f9cac7e0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:35:44.811+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862944:811116][50166:0x7f9cac7e0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:35:44.916+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862944:916444][50166:0x7f9cac7e0bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T00:35:45.031+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862945:31942][50166:0x7f9cac7e0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:35:45.111+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862945:111231][50166:0x7f9cac7e0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:35:45.181+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862945:181897][50166:0x7f9cac7e0bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:35:45.181+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862945:181982][50166:0x7f9cac7e0bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:45.235+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:35:45.235] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:56063 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:35:45.235] Error starting mongod : [00:35:45.235] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56063,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56063 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:35:44.036Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 50166 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:44.061+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:44.061+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:44.061+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56063,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:44.063+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:44.065+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:44.067+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:44.113+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:44.113+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:44.114+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:44.160+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:44.160+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:44.696+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862944:696767][50166:0x7f9cac7e0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:35:44.811+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:45.236+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:35:45.236] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:35:45.235] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:56063 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:35:45.235] Error starting mongod : [00:35:45.235] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56063,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56063 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:35:44.036Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 50166 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:44.061+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:44.061+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:44.061+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56063,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:44.063+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:44.065+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:44.067+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:44.113+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:44.113+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:44.114+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:44.160+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:44.160+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:44.696+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862944:696767][50166:0x7f9cac7e0bc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:45.236+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:35:45.236] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:35:45.236+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:35:45.236] Initiating stopEphemeralMongo [2021-10-22T00:35:45.236+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:35:45.236] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:35:45.236+0000] [.error] [src/director/director.go:executePlan:962] [00:35:45.236] Failed to apply action. Result = : [00:35:45.236] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:35:45.235] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:56063 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:35:45.235] Error starting mongod : [00:35:45.235] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56063,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56063 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:35:44.036Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 50166 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:44.061+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:44.061+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:44.061+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56063,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:44.063+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:44.065+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:44.067+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:44.113+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:44.113+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:44.114+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:44.160+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:44.160+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:44.696+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:45.236+0000] [.error] [src/director/director.go:planAndExecute:567] [00:35:45.236] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:35:45.236] Failed to apply action. Result = : [00:35:45.236] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:35:45.235] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:56063 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:35:45.235] Error starting mongod : [00:35:45.235] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56063,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56063 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:35:44.036Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 50166 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:44.061+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:44.061+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:44.061+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56063,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:44.063+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:44.065+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:44.067+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:44.113+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:44.113+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:44.114+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:44.160+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:44.160+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:45.237+0000] [.error] [src/director/director.go:mainLoop:399] [00:35:45.237] Failed to planAndExecute : [00:35:45.236] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:35:45.236] Failed to apply action. Result = : [00:35:45.236] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:35:45.235] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:56063 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:35:45.235] Error starting mongod : [00:35:45.235] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56063,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56063 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:35:44.036Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 50166 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:44.061+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:44.061+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:44.061+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56063,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:44.063+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:44.065+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:44.067+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:44.113+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:44.113+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:44.114+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:44.160+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:44.160+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:35:45.465+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:35:45.465] clusterConfig unchanged [2021-10-22T00:35:45.984+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:45.984] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:46.974+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:46.974] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:47.974+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:47.974] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:48.976+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:48.976] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:49.977+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:49.977] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:50.978+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:50.978] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:51.819+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:35:51.819] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 21.938739063s [2021-10-22T00:35:51.979+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:51.979] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:52.993+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:52.993] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:53.981+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:53.981] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:54.983+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:54.983] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:55.568+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:35:55.568] clusterConfig unchanged [2021-10-22T00:35:55.983+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:55.983] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:56.448+0000] [.info] [src/director/director.go:computePlan:280] [00:35:56.448] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:35:56.448+0000] [.info] [src/director/director.go:tracef:794] [00:35:56.448] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:56.448+0000] [.info] [src/director/director.go:tracef:794] [00:35:56.448] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:35:56.449+0000] [.info] [src/director/director.go:tracef:794] [00:35:56.449] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:56.449+0000] [.info] [src/director/director.go:tracef:794] [00:35:56.449] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:35:56.449+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:35:56.449] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:35:56.651+0000] [.info] [src/director/director.go:tracef:794] [00:35:56.651] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:56.651+0000] [.info] [src/director/director.go:tracef:794] [00:35:56.651] because [2021-10-22T00:35:56.652+0000] [.info] [src/director/director.go:tracef:794] [00:35:56.652] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:56.652+0000] [.info] [src/director/director.go:tracef:794] [00:35:56.652] because ['currentState.CanBounceStop' = true] [2021-10-22T00:35:56.857+0000] [.info] [src/director/director.go:tracef:794] [00:35:56.857] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:56.857+0000] [.info] [src/director/director.go:tracef:794] [00:35:56.857] because [2021-10-22T00:35:56.857+0000] [.info] [src/director/director.go:tracef:794] [00:35:56.857] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:56.857+0000] [.info] [src/director/director.go:tracef:794] [00:35:56.857] because ['currentState.Up' = false] [2021-10-22T00:35:56.858+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:35:56.857] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:35:56.858+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:35:56.858] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:35:56.946+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:35:56.946] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:35:56.958+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:35:56.958] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:35:56.959+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:35:56.959] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:35:56.961+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:35:56.961] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:35:56.962+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:35:56.962] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:35:56.962+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:35:56.962] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:35:56.966+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:35:56.966] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:35:56.985+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:56.985] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:57.221+0000] [.info] [src/director/director.go:tracef:794] [00:35:57.221] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:57.221+0000] [.info] [src/director/director.go:tracef:794] [00:35:57.221] because ['currentState.Up' = false] [2021-10-22T00:35:57.446+0000] [.info] [src/director/director.go:tracef:794] [00:35:57.446] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:57.446+0000] [.info] [src/director/director.go:tracef:794] [00:35:57.446] because ['currentState.Up' = false] [2021-10-22T00:35:57.446+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:35:57.446] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:35:57.684+0000] [.info] [src/director/director.go:tracef:794] [00:35:57.684] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:35:57.685+0000] [.info] [src/director/director.go:tracef:794] [00:35:57.684] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:35:57.950+0000] [.info] [src/director/director.go:tracef:794] [00:35:57.950] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:35:57.951+0000] [.info] [src/director/director.go:tracef:794] [00:35:57.951] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:35:57.951+0000] [.info] [src/director/director.go:tracef:794] [00:35:57.951] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:35:57.951+0000] [.info] [src/director/director.go:tracef:794] [00:35:57.951] because ['currentState.Up' = false] [2021-10-22T00:35:57.951+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:35:57.951] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:35:57.951+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:35:57.951] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:35:57.983+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:57.983] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:57.987+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:35:57.987] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:35:57.987+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:35:57.987] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36562 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:35:58.420+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:35:58.420] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 16.974149627s [2021-10-22T00:35:58.721+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:35:58.721] ...Finished start [pid=4205] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36562,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:35:58.852+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:35:58.852] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 131.429541ms. [2021-10-22T00:35:58.852+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:35:58.852] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36562 (local=false) [2021-10-22T00:35:58.852+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:35:58.852] Dropping existing documents from system.replset [2021-10-22T00:35:58.983+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:58.983] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:35:58.987+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:35:58.987] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:35:58.992+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:35:58.991] Inserting oplogTruncateAfterPoint document [2021-10-22T00:35:58.992+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:35:58.992] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:35:58.993+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:35:58.993] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:35:58.993+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:35:58.993] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:35:59.007+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:35:59.007] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:35:59.007+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:35:59.007] Setting the minvalid document to [{_id ObjectID("6172076f523370002becc26a")} {t -1} {ts {0 1}}] [2021-10-22T00:35:59.020+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:35:59.020] Inserted [{_id ObjectID("6172076f523370002becc26a")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:35:59.020+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:35:59.020] Inserting new replica set configuration document. [2021-10-22T00:35:59.020+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:35:59.020] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:35:59.034+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:35:59.034] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:35:59.038+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:35:59.038] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:35:59.038+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:35:59.038] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:35:59.038+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:35:59.038] Initiating stopEphemeralMongo [2021-10-22T00:35:59.038+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:35:59.038] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36562 (local=false) [2021-10-22T00:35:59.038+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:35:59.038] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36562 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:35:59.466+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:35:59.466] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.0544ms. [2021-10-22T00:35:59.466+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:35:59.466] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36562 (local=false) [2021-10-22T00:35:59.466+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:35:59.466] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.032µs. [2021-10-22T00:35:59.466+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:35:59.466] Successfully finished running stopEphemeralMongo [2021-10-22T00:35:59.466+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:35:59.466] Successfully stopped existing ephemeral mongod. [2021-10-22T00:35:59.466+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:35:59.466] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:35:59.466+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:35:59.466] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:35:59.500+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:35:59.500] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:35:59.500+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:35:59.500] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36086 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:35:59.986+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:35:59.986] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:00.681+0000] [.error] [src/action/start.go:func1:93] [103] [00:36:00.681] Error starting mongod : [00:36:00.681] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36086,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36086 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:35:59.529Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 5808 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:59.550+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:59.550+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:59.550+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36086,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:59.552+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:59.554+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:59.564+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:59.583+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:59.583+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:59.590+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:59.635+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:59.635+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:00.164+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862960:164482][5808:0x7f3f03e4abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:36:00.270+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862960:270045][5808:0x7f3f03e4abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:36:00.371+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862960:371092][5808:0x7f3f03e4abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:36:00.486+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862960:486972][5808:0x7f3f03e4abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:36:00.561+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862960:561237][5808:0x7f3f03e4abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:36:00.616+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862960:616550][5808:0x7f3f03e4abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:36:00.616+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862960:616622][5808:0x7f3f03e4abc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:00.681+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:36:00.681] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36086 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:36:00.681] Error starting mongod : [00:36:00.681] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36086,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36086 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:35:59.529Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 5808 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:59.550+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:59.550+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:59.550+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36086,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:59.552+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:59.554+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:59.564+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:59.583+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:59.583+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:59.590+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:59.635+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:59.635+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:00.164+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862960:164482][5808:0x7f3f03e4abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:36:00.270+00:00"},"s...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:00.681+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:36:00.681] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:36:00.681] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36086 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:36:00.681] Error starting mongod : [00:36:00.681] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36086,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36086 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:35:59.529Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 5808 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:59.550+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:59.550+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:59.550+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36086,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:59.552+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:59.554+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:59.564+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:59.583+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:59.583+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:59.590+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:59.635+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:59.635+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:00.164+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862960:164482][5808:0x7f3f03e4abc0... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:00.682+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:36:00.682] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:36:00.682+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:36:00.682] Initiating stopEphemeralMongo [2021-10-22T00:36:00.682+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:36:00.682] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:36:00.682+0000] [.error] [src/director/director.go:executePlan:962] [00:36:00.682] Failed to apply action. Result = : [00:36:00.681] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:36:00.681] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36086 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:36:00.681] Error starting mongod : [00:36:00.681] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36086,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36086 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:35:59.529Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 5808 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:59.550+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:59.550+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:59.550+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36086,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:59.552+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:59.554+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:59.564+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:59.583+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:59.583+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:59.590+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:59.635+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:59.635+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:00.164+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:00.682+0000] [.error] [src/director/director.go:planAndExecute:567] [00:36:00.682] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:36:00.682] Failed to apply action. Result = : [00:36:00.681] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:36:00.681] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36086 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:36:00.681] Error starting mongod : [00:36:00.681] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36086,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36086 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:35:59.529Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 5808 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:59.550+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:59.550+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:59.550+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36086,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:59.552+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:59.554+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:59.564+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:59.583+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:59.583+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:59.590+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:59.635+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:59.635+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:00.682+0000] [.error] [src/director/director.go:mainLoop:399] [00:36:00.682] Failed to planAndExecute : [00:36:00.682] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:36:00.682] Failed to apply action. Result = : [00:36:00.681] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:36:00.681] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36086 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:36:00.681] Error starting mongod : [00:36:00.681] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36086,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36086 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:35:59.529Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 5808 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:35:59.550+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:35:59.550+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:35:59.550+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36086,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:35:59.552+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:59.554+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:35:59.564+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:35:59.583+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:59.583+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:35:59.590+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:35:59.635+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:35:59.635+00:00"},"s":"I", "c":"NETWORK", "id"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:00.988+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:00.988] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:01.986+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:01.986] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:02.986+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:02.986] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:03.987+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:03.987] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:04.989+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:04.989] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:05.817+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:36:05.817] clusterConfig unchanged [2021-10-22T00:36:05.987+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:05.987] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:06.989+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:06.989] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:07.990+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:07.990] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:08.992+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:08.992] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:09.645+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:36:09.645] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.679261144s [2021-10-22T00:36:09.990+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:09.990] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:10.990+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:10.990] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:11.852+0000] [.info] [src/director/director.go:computePlan:280] [00:36:11.852] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:36:11.852+0000] [.info] [src/director/director.go:tracef:794] [00:36:11.852] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:11.852+0000] [.info] [src/director/director.go:tracef:794] [00:36:11.852] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:36:11.852+0000] [.info] [src/director/director.go:tracef:794] [00:36:11.852] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:11.852+0000] [.info] [src/director/director.go:tracef:794] [00:36:11.852] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:36:11.853+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:36:11.853] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:36:11.996+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:11.996] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:12.038+0000] [.info] [src/director/director.go:tracef:794] [00:36:12.038] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:12.038+0000] [.info] [src/director/director.go:tracef:794] [00:36:12.038] because [2021-10-22T00:36:12.039+0000] [.info] [src/director/director.go:tracef:794] [00:36:12.039] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:12.039+0000] [.info] [src/director/director.go:tracef:794] [00:36:12.039] because ['currentState.CanBounceStop' = true] [2021-10-22T00:36:12.191+0000] [.info] [src/director/director.go:tracef:794] [00:36:12.191] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:12.192+0000] [.info] [src/director/director.go:tracef:794] [00:36:12.192] because [2021-10-22T00:36:12.192+0000] [.info] [src/director/director.go:tracef:794] [00:36:12.192] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:12.192+0000] [.info] [src/director/director.go:tracef:794] [00:36:12.192] because ['currentState.Up' = false] [2021-10-22T00:36:12.192+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:36:12.192] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:36:12.192+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:36:12.192] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:36:12.281+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:36:12.281] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:36:12.289+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:36:12.289] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:36:12.289+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:36:12.289] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:36:12.290+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:36:12.290] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:36:12.291+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:36:12.291] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:36:12.291+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:36:12.291] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:36:12.291+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:36:12.291] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:36:12.292+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:36:12.292] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:36:12.496+0000] [.info] [src/director/director.go:tracef:794] [00:36:12.496] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:12.496+0000] [.info] [src/director/director.go:tracef:794] [00:36:12.496] because ['currentState.Up' = false] [2021-10-22T00:36:12.689+0000] [.info] [src/director/director.go:tracef:794] [00:36:12.689] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:12.689+0000] [.info] [src/director/director.go:tracef:794] [00:36:12.689] because ['currentState.Up' = false] [2021-10-22T00:36:12.689+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:36:12.689] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:36:12.898+0000] [.info] [src/director/director.go:tracef:794] [00:36:12.898] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:12.898+0000] [.info] [src/director/director.go:tracef:794] [00:36:12.898] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:36:12.996+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:12.996] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:13.118+0000] [.info] [src/director/director.go:tracef:794] [00:36:13.118] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:36:13.118+0000] [.info] [src/director/director.go:tracef:794] [00:36:13.118] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:36:13.119+0000] [.info] [src/director/director.go:tracef:794] [00:36:13.119] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:36:13.119+0000] [.info] [src/director/director.go:tracef:794] [00:36:13.119] because ['currentState.Up' = false] [2021-10-22T00:36:13.119+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:36:13.119] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:36:13.119+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:36:13.119] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:36:13.154+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:36:13.154] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:36:13.155+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:36:13.155] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47427 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:36:13.875+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:36:13.875] ...Finished start [pid=25355] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47427,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:36:13.996+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:13.996] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:14.007+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:36:14.007] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 131.96797ms. [2021-10-22T00:36:14.007+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:36:14.007] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47427 (local=false) [2021-10-22T00:36:14.008+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:36:14.008] Dropping existing documents from system.replset [2021-10-22T00:36:14.127+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:36:14.127] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:36:14.132+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:36:14.132] Inserting oplogTruncateAfterPoint document [2021-10-22T00:36:14.132+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:36:14.132] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:36:14.134+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:36:14.134] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:36:14.134+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:36:14.134] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:36:14.148+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:36:14.148] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:36:14.148+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:36:14.148] Setting the minvalid document to [{_id ObjectID("6172077e523370002becc277")} {t -1} {ts {0 1}}] [2021-10-22T00:36:14.163+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:36:14.163] Inserted [{_id ObjectID("6172077e523370002becc277")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:36:14.163+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:36:14.163] Inserting new replica set configuration document. [2021-10-22T00:36:14.163+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:36:14.163] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:36:14.176+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:36:14.176] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:36:14.190+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:36:14.190] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:36:14.190+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:36:14.190] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:36:14.190+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:36:14.190] Initiating stopEphemeralMongo [2021-10-22T00:36:14.190+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:36:14.190] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47427 (local=false) [2021-10-22T00:36:14.190+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:36:14.190] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47427 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:36:14.678+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:36:14.678] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.304906ms. [2021-10-22T00:36:14.678+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:36:14.678] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47427 (local=false) [2021-10-22T00:36:14.678+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:36:14.678] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.493µs. [2021-10-22T00:36:14.679+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:36:14.678] Successfully finished running stopEphemeralMongo [2021-10-22T00:36:14.679+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:36:14.679] Successfully stopped existing ephemeral mongod. [2021-10-22T00:36:14.679+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:36:14.679] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:36:14.679+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:36:14.679] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:36:14.715+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:36:14.715] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:36:14.716+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:36:14.716] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40599 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:36:15.008+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:15.008] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:15.898+0000] [.error] [src/action/start.go:func1:93] [103] [00:36:15.898] Error starting mongod : [00:36:15.898] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40599 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 26937 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:36:14.770+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:36:14.771+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:36:14.771+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:36:14.772+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:14.774+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:36:14.775+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:36:14.812+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:14.812+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:14.812+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:14.857+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:14.857+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:15.383+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862975:383355][26937:0x7f6e42ec1bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:36:15.488+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862975:488910][26937:0x7f6e42ec1bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:36:15.586+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862975:586884][26937:0x7f6e42ec1bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:36:15.699+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862975:699920][26937:0x7f6e42ec1bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:36:15.780+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862975:780544][26937:0x7f6e42ec1bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:36:15.845+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862975:845207][26937:0x7f6e42ec1bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:36:15.845+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862975:845280][26937:0x7f6e42ec1bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:36:15.846+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:15.899+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:36:15.899] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:40599 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:36:15.898] Error starting mongod : [00:36:15.898] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40599 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 26937 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:36:14.770+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:36:14.771+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:36:14.771+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:36:14.772+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:14.774+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:36:14.775+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:36:14.812+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:14.812+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:14.812+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:14.857+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:14.857+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:15.383+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862975:383355][26937:0x7f6e42ec1bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:36:15.488+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862975:488910][26937:0x7f6e42ec1bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:15.900+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:36:15.900] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:36:15.899] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:40599 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:36:15.898] Error starting mongod : [00:36:15.898] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40599 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 26937 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:36:14.770+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:36:14.771+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:36:14.771+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:36:14.772+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:14.774+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:36:14.775+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:36:14.812+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:14.812+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:14.812+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:14.857+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:14.857+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:15.383+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862975:383355][26937:0x7f6e42ec1bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:36:15.488+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:15.900+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:36:15.900] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:36:15.900+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:36:15.900] Initiating stopEphemeralMongo [2021-10-22T00:36:15.900+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:36:15.900] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:36:15.906+0000] [.error] [src/director/director.go:executePlan:962] [00:36:15.906] Failed to apply action. Result = : [00:36:15.900] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:36:15.899] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:40599 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:36:15.898] Error starting mongod : [00:36:15.898] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40599 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 26937 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:36:14.770+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:36:14.771+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:36:14.771+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:36:14.772+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:14.774+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:36:14.775+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:36:14.812+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:14.812+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:14.812+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:14.857+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:14.857+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:15.383+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862975:383355][26937:0x7f6e42ec1bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:15.906+0000] [.error] [src/director/director.go:planAndExecute:567] [00:36:15.906] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:36:15.906] Failed to apply action. Result = : [00:36:15.900] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:36:15.899] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:40599 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:36:15.898] Error starting mongod : [00:36:15.898] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40599 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 26937 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:36:14.770+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:36:14.771+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:36:14.771+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:36:14.772+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:14.774+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:36:14.775+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:36:14.812+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:14.812+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:14.812+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:14.857+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:14.857+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:15.383+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:15.906+0000] [.error] [src/director/director.go:mainLoop:399] [00:36:15.906] Failed to planAndExecute : [00:36:15.906] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:36:15.906] Failed to apply action. Result = : [00:36:15.900] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:36:15.899] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:40599 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:36:15.898] Error starting mongod : [00:36:15.898] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40599 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 26937 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:36:14.770+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:36:14.771+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:36:14.771+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:36:14.772+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:14.774+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:36:14.775+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:36:14.812+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:14.812+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:14.812+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:14.857+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:14.857+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:15.383+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:15.997+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:36:15.997] clusterConfig unchanged [2021-10-22T00:36:15.999+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:15.999] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:16.997+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:16.997] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:17.267+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:36:17.267] clusterConfig unchanged [2021-10-22T00:36:18.000+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:18.000] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:19.001+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:19.001] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:19.999+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:19.999] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:21.005+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:21.005] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:22.014+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:22.014] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:23.003+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:23.003] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:24.003+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:24.003] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:24.584+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:36:24.584] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.29180242s [2021-10-22T00:36:25.001+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:25.001] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:26.002+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:26.002] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:27.002+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:27.002] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:27.379+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:36:27.379] clusterConfig unchanged [2021-10-22T00:36:28.011+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:28.011] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:29.004+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:29.004] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:30.004+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:30.004] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:31.005+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:31.005] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:32.007+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:32.007] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:32.079+0000] [.info] [src/director/director.go:computePlan:280] [00:36:32.079] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:36:32.079+0000] [.info] [src/director/director.go:tracef:794] [00:36:32.079] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:32.079+0000] [.info] [src/director/director.go:tracef:794] [00:36:32.079] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:36:32.079+0000] [.info] [src/director/director.go:tracef:794] [00:36:32.079] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:32.079+0000] [.info] [src/director/director.go:tracef:794] [00:36:32.079] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:36:32.079+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:36:32.079] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:36:32.238+0000] [.info] [src/director/director.go:tracef:794] [00:36:32.238] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:32.238+0000] [.info] [src/director/director.go:tracef:794] [00:36:32.238] because [2021-10-22T00:36:32.238+0000] [.info] [src/director/director.go:tracef:794] [00:36:32.238] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:32.238+0000] [.info] [src/director/director.go:tracef:794] [00:36:32.238] because ['currentState.CanBounceStop' = true] [2021-10-22T00:36:32.393+0000] [.info] [src/director/director.go:tracef:794] [00:36:32.393] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:32.393+0000] [.info] [src/director/director.go:tracef:794] [00:36:32.393] because [2021-10-22T00:36:32.393+0000] [.info] [src/director/director.go:tracef:794] [00:36:32.393] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:32.393+0000] [.info] [src/director/director.go:tracef:794] [00:36:32.393] because ['currentState.Up' = false] [2021-10-22T00:36:32.393+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:36:32.393] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:36:32.393+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:36:32.393] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:36:32.480+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:36:32.480] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:36:32.488+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:36:32.488] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:36:32.489+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:36:32.489] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:36:32.489+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:36:32.489] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:36:32.490+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:36:32.490] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:36:32.491+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:36:32.491] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:36:32.491+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:36:32.491] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:36:32.492+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:36:32.492] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:36:32.717+0000] [.info] [src/director/director.go:tracef:794] [00:36:32.717] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:32.717+0000] [.info] [src/director/director.go:tracef:794] [00:36:32.717] because ['currentState.Up' = false] [2021-10-22T00:36:32.927+0000] [.info] [src/director/director.go:tracef:794] [00:36:32.927] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:32.927+0000] [.info] [src/director/director.go:tracef:794] [00:36:32.927] because ['currentState.Up' = false] [2021-10-22T00:36:32.927+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:36:32.927] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:36:33.038+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:33.038] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:33.153+0000] [.info] [src/director/director.go:tracef:794] [00:36:33.153] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:33.153+0000] [.info] [src/director/director.go:tracef:794] [00:36:33.153] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:36:33.354+0000] [.info] [src/director/director.go:tracef:794] [00:36:33.354] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:36:33.354+0000] [.info] [src/director/director.go:tracef:794] [00:36:33.354] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:36:33.355+0000] [.info] [src/director/director.go:tracef:794] [00:36:33.355] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:36:33.355+0000] [.info] [src/director/director.go:tracef:794] [00:36:33.355] because ['currentState.Up' = false] [2021-10-22T00:36:33.355+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:36:33.355] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:36:33.355+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:36:33.355] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:36:33.389+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:36:33.389] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:36:33.390+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:36:33.390] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50287 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:36:34.009+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:34.009] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:34.101+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:36:34.101] ...Finished start [pid=46188] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50287,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:36:34.216+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:36:34.216] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 114.763151ms. [2021-10-22T00:36:34.216+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:36:34.216] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50287 (local=false) [2021-10-22T00:36:34.216+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:36:34.216] Dropping existing documents from system.replset [2021-10-22T00:36:34.337+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:36:34.337] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:36:34.352+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:36:34.352] Inserting oplogTruncateAfterPoint document [2021-10-22T00:36:34.352+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:36:34.352] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:36:34.354+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:36:34.354] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:36:34.354+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:36:34.354] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:36:34.368+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:36:34.368] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:36:34.368+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:36:34.368] Setting the minvalid document to [{_id ObjectID("61720792523370002becc284")} {t -1} {ts {0 1}}] [2021-10-22T00:36:34.381+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:36:34.381] Inserted [{_id ObjectID("61720792523370002becc284")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:36:34.381+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:36:34.381] Inserting new replica set configuration document. [2021-10-22T00:36:34.381+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:36:34.381] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:36:34.394+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:36:34.394] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:36:34.399+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:36:34.399] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:36:34.399+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:36:34.399] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:36:34.399+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:36:34.399] Initiating stopEphemeralMongo [2021-10-22T00:36:34.399+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:36:34.399] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50287 (local=false) [2021-10-22T00:36:34.399+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:36:34.399] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50287 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:36:34.767+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:36:34.767] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.329843ms. [2021-10-22T00:36:34.767+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:36:34.767] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50287 (local=false) [2021-10-22T00:36:34.767+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:36:34.767] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.126µs. [2021-10-22T00:36:34.767+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:36:34.767] Successfully finished running stopEphemeralMongo [2021-10-22T00:36:34.767+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:36:34.767] Successfully stopped existing ephemeral mongod. [2021-10-22T00:36:34.767+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:36:34.767] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:36:34.767+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:36:34.767] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:36:34.805+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:36:34.805] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:36:34.806+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:36:34.806] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50750 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:36:35.011+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:35.011] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:35.953+0000] [.error] [src/action/start.go:func1:93] [103] [00:36:35.953] Error starting mongod : [00:36:35.953] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50750,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50750 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:36:34.838Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 47766 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:36:34.860+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:36:34.860+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:36:34.860+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50750,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:36:34.861+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:34.863+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:36:34.865+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:36:34.901+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:34.901+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:34.901+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:34.947+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:34.947+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:35.454+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862995:454349][47766:0x7faf3b340bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:36:35.562+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862995:562405][47766:0x7faf3b340bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:36:35.664+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862995:664318][47766:0x7faf3b340bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:36:35.779+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862995:779094][47766:0x7faf3b340bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:36:35.849+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862995:849661][47766:0x7faf3b340bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:36:35.905+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862995:905018][47766:0x7faf3b340bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:36:35.905+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862995:905091][47766:0x7faf3b340bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:35.953+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:36:35.953] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:50750 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:36:35.953] Error starting mongod : [00:36:35.953] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50750,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50750 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:36:34.838Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 47766 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:36:34.860+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:36:34.860+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:36:34.860+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50750,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:36:34.861+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:34.863+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:36:34.865+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:36:34.901+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:34.901+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:34.901+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:34.947+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:34.947+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:35.454+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862995:454349][47766:0x7faf3b340bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:36:35.562+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:35.953+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:36:35.953] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:36:35.953] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:50750 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:36:35.953] Error starting mongod : [00:36:35.953] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50750,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50750 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:36:34.838Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 47766 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:36:34.860+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:36:34.860+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:36:34.860+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50750,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:36:34.861+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:34.863+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:36:34.865+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:36:34.901+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:34.901+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:34.901+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:34.947+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:34.947+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:35.454+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634862995:454349][47766:0x7faf3b340bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:35.954+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:36:35.954] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:36:35.954+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:36:35.954] Initiating stopEphemeralMongo [2021-10-22T00:36:35.954+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:36:35.954] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:36:35.954+0000] [.error] [src/director/director.go:executePlan:962] [00:36:35.954] Failed to apply action. Result = : [00:36:35.953] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:36:35.953] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:50750 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:36:35.953] Error starting mongod : [00:36:35.953] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50750,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50750 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:36:34.838Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 47766 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:36:34.860+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:36:34.860+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:36:34.860+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50750,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:36:34.861+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:34.863+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:36:34.865+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:36:34.901+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:34.901+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:34.901+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:34.947+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:34.947+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:35.454+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:35.954+0000] [.error] [src/director/director.go:planAndExecute:567] [00:36:35.954] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:36:35.954] Failed to apply action. Result = : [00:36:35.953] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:36:35.953] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:50750 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:36:35.953] Error starting mongod : [00:36:35.953] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50750,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50750 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:36:34.838Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 47766 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:36:34.860+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:36:34.860+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:36:34.860+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50750,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:36:34.861+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:34.863+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:36:34.865+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:36:34.901+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:34.901+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:34.901+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:34.947+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:34.947+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:35.954+0000] [.error] [src/director/director.go:mainLoop:399] [00:36:35.954] Failed to planAndExecute : [00:36:35.954] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:36:35.954] Failed to apply action. Result = : [00:36:35.953] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:36:35.953] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:50750 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:36:35.953] Error starting mongod : [00:36:35.953] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50750,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50750 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:36:34.838Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 47766 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:36:34.860+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:36:34.860+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:36:34.860+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50750,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:36:34.861+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:34.863+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:36:34.865+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:36:34.901+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:34.901+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:34.901+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:34.947+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:34.947+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:36.011+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:36.011] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:37.011+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:37.011] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:37.639+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:36:37.639] clusterConfig unchanged [2021-10-22T00:36:38.011+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:38.011] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:39.013+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:39.013] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:40.011+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:40.011] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:41.013+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:41.013] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:41.257+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:36:41.257] memory:rss=49131520 alloc=11026936 totalalloc=1218039376 mallocs=16158252 frees=16077834 heapinuse=14958592 heapobjects=80418 gcpausetotalns=1377094727 gcpauselastns=101786 gcnum=195 utimens=21068332000 stimens=5742974000 minflt=59623 majflt=404 [2021-10-22T00:36:42.014+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:42.014] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:43.015+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:43.015] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:44.014+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:44.014] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:44.597+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:36:44.597] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.105096267s [2021-10-22T00:36:45.014+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:45.014] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:46.016+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:46.016] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:47.016+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:47.016] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:47.781+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:36:47.781] clusterConfig unchanged [2021-10-22T00:36:48.019+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:48.019] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:49.018+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:49.018] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:50.018+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:50.018] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:51.019+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:51.019] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:52.019+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:52.019] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:52.124+0000] [.info] [src/director/director.go:computePlan:280] [00:36:52.124] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:36:52.125+0000] [.info] [src/director/director.go:tracef:794] [00:36:52.125] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:52.125+0000] [.info] [src/director/director.go:tracef:794] [00:36:52.125] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:36:52.125+0000] [.info] [src/director/director.go:tracef:794] [00:36:52.125] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:52.125+0000] [.info] [src/director/director.go:tracef:794] [00:36:52.125] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:36:52.125+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:36:52.125] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:36:52.272+0000] [.info] [src/director/director.go:tracef:794] [00:36:52.272] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:52.272+0000] [.info] [src/director/director.go:tracef:794] [00:36:52.272] because [2021-10-22T00:36:52.273+0000] [.info] [src/director/director.go:tracef:794] [00:36:52.273] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:52.273+0000] [.info] [src/director/director.go:tracef:794] [00:36:52.273] because ['currentState.CanBounceStop' = true] [2021-10-22T00:36:52.417+0000] [.info] [src/director/director.go:tracef:794] [00:36:52.417] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:52.417+0000] [.info] [src/director/director.go:tracef:794] [00:36:52.417] because [2021-10-22T00:36:52.417+0000] [.info] [src/director/director.go:tracef:794] [00:36:52.417] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:52.418+0000] [.info] [src/director/director.go:tracef:794] [00:36:52.417] because ['currentState.Up' = false] [2021-10-22T00:36:52.418+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:36:52.418] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:36:52.418+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:36:52.418] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:36:52.497+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:36:52.497] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:36:52.506+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:36:52.506] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:36:52.508+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:36:52.508] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:36:52.510+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:36:52.510] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:36:52.510+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:36:52.510] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:36:52.512+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:36:52.512] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:36:52.514+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:36:52.514] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:36:52.722+0000] [.info] [src/director/director.go:tracef:794] [00:36:52.722] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:52.722+0000] [.info] [src/director/director.go:tracef:794] [00:36:52.722] because ['currentState.Up' = false] [2021-10-22T00:36:52.930+0000] [.info] [src/director/director.go:tracef:794] [00:36:52.930] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:52.930+0000] [.info] [src/director/director.go:tracef:794] [00:36:52.930] because ['currentState.Up' = false] [2021-10-22T00:36:52.930+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:36:52.930] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:36:53.040+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:53.040] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:53.166+0000] [.info] [src/director/director.go:tracef:794] [00:36:53.166] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:36:53.166+0000] [.info] [src/director/director.go:tracef:794] [00:36:53.166] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:36:53.364+0000] [.info] [src/director/director.go:tracef:794] [00:36:53.364] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:36:53.365+0000] [.info] [src/director/director.go:tracef:794] [00:36:53.365] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:36:53.365+0000] [.info] [src/director/director.go:tracef:794] [00:36:53.365] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:36:53.365+0000] [.info] [src/director/director.go:tracef:794] [00:36:53.365] because ['currentState.Up' = false] [2021-10-22T00:36:53.365+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:36:53.365] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:36:53.365+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:36:53.365] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:36:53.398+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:36:53.398] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:36:53.398+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:36:53.398] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36433 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:36:54.036+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:54.036] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:54.117+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:36:54.117] ...Finished start [pid=66970] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36433,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:36:54.309+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:36:54.309] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 191.423626ms. [2021-10-22T00:36:54.309+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:36:54.309] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36433 (local=false) [2021-10-22T00:36:54.309+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:36:54.309] Dropping existing documents from system.replset [2021-10-22T00:36:54.456+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:36:54.456] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:36:54.460+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:36:54.460] Inserting oplogTruncateAfterPoint document [2021-10-22T00:36:54.460+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:36:54.460] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:36:54.461+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:36:54.461] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:36:54.461+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:36:54.461] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:36:54.474+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:36:54.474] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:36:54.474+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:36:54.474] Setting the minvalid document to [{_id ObjectID("617207a6523370002becc292")} {t -1} {ts {0 1}}] [2021-10-22T00:36:54.487+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:36:54.487] Inserted [{_id ObjectID("617207a6523370002becc292")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:36:54.487+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:36:54.487] Inserting new replica set configuration document. [2021-10-22T00:36:54.487+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:36:54.487] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:36:54.498+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:36:54.498] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:36:54.522+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:36:54.522] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:36:54.522+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:36:54.522] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:36:54.522+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:36:54.522] Initiating stopEphemeralMongo [2021-10-22T00:36:54.522+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:36:54.522] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36433 (local=false) [2021-10-22T00:36:54.522+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:36:54.522] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36433 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:36:55.019+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:36:55.019] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.163052ms. [2021-10-22T00:36:55.019+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:36:55.019] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36433 (local=false) [2021-10-22T00:36:55.019+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:36:55.019] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.256µs. [2021-10-22T00:36:55.020+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:36:55.020] Successfully finished running stopEphemeralMongo [2021-10-22T00:36:55.020+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:36:55.020] Successfully stopped existing ephemeral mongod. [2021-10-22T00:36:55.020+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:36:55.020] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:36:55.020+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:36:55.020] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:36:55.051+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:55.051] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:55.066+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:36:55.066] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:36:55.067+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:36:55.067] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39065 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:36:56.054+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:56.054] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:56.216+0000] [.error] [src/action/start.go:func1:93] [103] [00:36:56.216] Error starting mongod : [00:36:56.215] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39065,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39065 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:36:55.096Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 68590 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:36:55.116+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:36:55.116+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:36:55.116+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39065,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:36:55.118+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:55.120+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:36:55.120+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:36:55.168+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:55.168+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:55.168+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:55.221+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:55.221+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:55.724+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863015:724928][68590:0x7fefdce8bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:36:55.825+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863015:825433][68590:0x7fefdce8bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:36:55.924+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863015:924123][68590:0x7fefdce8bbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35456 to 2/256"}} {"t":{"$date":"2021-10-22T00:36:56.035+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863016:35949][68590:0x7fefdce8bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:36:56.109+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863016:109151][68590:0x7fefdce8bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:36:56.166+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863016:166412][68590:0x7fefdce8bbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:36:56.166+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863016:166487][68590:0x7fefdce8bbc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:56.216+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:36:56.216] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39065 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:36:56.216] Error starting mongod : [00:36:56.215] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39065,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39065 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:36:55.096Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 68590 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:36:55.116+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:36:55.116+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:36:55.116+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39065,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:36:55.118+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:55.120+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:36:55.120+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:36:55.168+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:55.168+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:55.168+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:55.221+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:55.221+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:55.724+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863015:724928][68590:0x7fefdce8bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:36:55.825+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:56.216+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:36:56.216] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:36:56.216] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39065 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:36:56.216] Error starting mongod : [00:36:56.215] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39065,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39065 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:36:55.096Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 68590 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:36:55.116+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:36:55.116+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:36:55.116+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39065,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:36:55.118+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:55.120+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:36:55.120+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:36:55.168+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:55.168+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:55.168+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:55.221+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:55.221+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:55.724+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863015:724928][68590:0x7fefdce8bbc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:56.216+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:36:56.216] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:36:56.216+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:36:56.216] Initiating stopEphemeralMongo [2021-10-22T00:36:56.216+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:36:56.216] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:36:56.217+0000] [.error] [src/director/director.go:executePlan:962] [00:36:56.217] Failed to apply action. Result = : [00:36:56.216] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:36:56.216] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39065 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:36:56.216] Error starting mongod : [00:36:56.215] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39065,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39065 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:36:55.096Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 68590 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:36:55.116+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:36:55.116+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:36:55.116+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39065,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:36:55.118+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:55.120+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:36:55.120+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:36:55.168+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:55.168+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:55.168+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:55.221+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:55.221+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:55.724+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:56.217+0000] [.error] [src/director/director.go:planAndExecute:567] [00:36:56.217] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:36:56.217] Failed to apply action. Result = : [00:36:56.216] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:36:56.216] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39065 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:36:56.216] Error starting mongod : [00:36:56.215] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39065,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39065 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:36:55.096Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 68590 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:36:55.116+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:36:55.116+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:36:55.116+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39065,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:36:55.118+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:55.120+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:36:55.120+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:36:55.168+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:55.168+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:55.168+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:55.221+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:55.221+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:56.217+0000] [.error] [src/director/director.go:mainLoop:399] [00:36:56.217] Failed to planAndExecute : [00:36:56.217] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:36:56.217] Failed to apply action. Result = : [00:36:56.216] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:36:56.216] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39065 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:36:56.216] Error starting mongod : [00:36:56.215] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39065,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39065 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:36:55.096Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 68590 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:36:55.116+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:36:55.116+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:36:55.116+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39065,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:36:55.118+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:55.120+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:36:55.120+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:36:55.168+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:55.168+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:36:55.168+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:36:55.221+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:36:55.221+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:36:57.054+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:57.054] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:58.030+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:36:58.030] clusterConfig unchanged [2021-10-22T00:36:58.053+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:58.053] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:36:59.054+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:36:59.054] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:00.055+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:00.055] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:01.055+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:01.055] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:02.054+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:02.054] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:03.059+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:03.059] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:04.055+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:04.055] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:04.743+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:37:04.743] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.229686121s [2021-10-22T00:37:05.056+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:05.056] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:06.056+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:06.056] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:07.056+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:07.056] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:08.060+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:08.060] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:08.144+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:37:08.144] clusterConfig unchanged [2021-10-22T00:37:09.057+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:09.057] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:10.058+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:10.058] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:11.058+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:11.058] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:12.059+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:12.059] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:13.059+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:13.059] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:14.060+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:14.060] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:15.060+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:15.060] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:16.061+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:16.061] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:17.061+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:17.061] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:17.379+0000] [.info] [src/director/director.go:computePlan:280] [00:37:17.378] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:37:17.379+0000] [.info] [src/director/director.go:tracef:794] [00:37:17.379] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:37:17.379+0000] [.info] [src/director/director.go:tracef:794] [00:37:17.379] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:37:17.379+0000] [.info] [src/director/director.go:tracef:794] [00:37:17.379] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:37:17.379+0000] [.info] [src/director/director.go:tracef:794] [00:37:17.379] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:37:17.379+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:37:17.379] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:37:17.529+0000] [.info] [src/director/director.go:tracef:794] [00:37:17.529] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:37:17.529+0000] [.info] [src/director/director.go:tracef:794] [00:37:17.529] because [2021-10-22T00:37:17.530+0000] [.info] [src/director/director.go:tracef:794] [00:37:17.530] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:37:17.530+0000] [.info] [src/director/director.go:tracef:794] [00:37:17.530] because ['currentState.CanBounceStop' = true] [2021-10-22T00:37:17.678+0000] [.info] [src/director/director.go:tracef:794] [00:37:17.678] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:37:17.678+0000] [.info] [src/director/director.go:tracef:794] [00:37:17.678] because [2021-10-22T00:37:17.679+0000] [.info] [src/director/director.go:tracef:794] [00:37:17.679] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:37:17.679+0000] [.info] [src/director/director.go:tracef:794] [00:37:17.679] because ['currentState.Up' = false] [2021-10-22T00:37:17.679+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:37:17.679] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:37:17.679+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:37:17.679] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:37:17.794+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:37:17.794] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:37:17.802+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:37:17.802] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:37:17.804+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:37:17.804] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:37:17.805+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:37:17.805] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:37:17.807+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:37:17.807] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:37:17.807+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:37:17.807] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:37:17.808+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:37:17.808] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:37:17.811+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:37:17.811] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:37:18.010+0000] [.info] [src/director/director.go:tracef:794] [00:37:18.010] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:37:18.011+0000] [.info] [src/director/director.go:tracef:794] [00:37:18.011] because ['currentState.Up' = false] [2021-10-22T00:37:18.065+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:18.065] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:18.238+0000] [.info] [src/director/director.go:tracef:794] [00:37:18.238] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:37:18.238+0000] [.info] [src/director/director.go:tracef:794] [00:37:18.238] because ['currentState.Up' = false] [2021-10-22T00:37:18.238+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:37:18.238] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:37:18.353+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:37:18.353] clusterConfig unchanged [2021-10-22T00:37:18.427+0000] [.info] [src/director/director.go:tracef:794] [00:37:18.427] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:37:18.427+0000] [.info] [src/director/director.go:tracef:794] [00:37:18.427] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:37:18.650+0000] [.info] [src/director/director.go:tracef:794] [00:37:18.650] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:37:18.650+0000] [.info] [src/director/director.go:tracef:794] [00:37:18.650] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:37:18.650+0000] [.info] [src/director/director.go:tracef:794] [00:37:18.650] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:37:18.650+0000] [.info] [src/director/director.go:tracef:794] [00:37:18.650] because ['currentState.Up' = false] [2021-10-22T00:37:18.650+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:37:18.650] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:37:18.650+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:37:18.650] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:37:18.686+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:37:18.686] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:37:18.687+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:37:18.687] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 54488 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:37:19.067+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:19.067] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:19.394+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:37:19.394] ...Finished start [pid=13677] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54488,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:37:19.472+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:37:19.472] clusterConfig unchanged [2021-10-22T00:37:19.507+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:37:19.507] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 112.420947ms. [2021-10-22T00:37:19.507+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:37:19.507] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:54488 (local=false) [2021-10-22T00:37:19.507+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:37:19.507] Dropping existing documents from system.replset [2021-10-22T00:37:19.631+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:37:19.631] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:37:19.635+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:37:19.635] Inserting oplogTruncateAfterPoint document [2021-10-22T00:37:19.635+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:37:19.635] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:37:19.636+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:37:19.636] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:37:19.636+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:37:19.636] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:37:19.648+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:37:19.648] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:37:19.648+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:37:19.648] Setting the minvalid document to [{_id ObjectID("617207bf523370002becc29f")} {t -1} {ts {0 1}}] [2021-10-22T00:37:19.661+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:37:19.661] Inserted [{_id ObjectID("617207bf523370002becc29f")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:37:19.661+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:37:19.661] Inserting new replica set configuration document. [2021-10-22T00:37:19.661+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:37:19.661] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:37:19.674+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:37:19.674] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:37:19.678+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:37:19.678] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:37:19.678+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:37:19.678] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:37:19.678+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:37:19.678] Initiating stopEphemeralMongo [2021-10-22T00:37:19.678+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:37:19.678] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:54488 (local=false) [2021-10-22T00:37:19.678+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:37:19.678] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:54488 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:37:20.067+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:20.067] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:20.110+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:37:20.110] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.122477ms. [2021-10-22T00:37:20.110+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:37:20.110] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:54488 (local=false) [2021-10-22T00:37:20.110+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:37:20.110] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.546µs. [2021-10-22T00:37:20.111+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:37:20.110] Successfully finished running stopEphemeralMongo [2021-10-22T00:37:20.111+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:37:20.111] Successfully stopped existing ephemeral mongod. [2021-10-22T00:37:20.111+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:37:20.111] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:37:20.111+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:37:20.111] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:37:20.148+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:37:20.148] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:37:20.149+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:37:20.148] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48711 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:37:20.568+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:37:20.568] clusterConfig unchanged [2021-10-22T00:37:21.070+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:21.070] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:21.311+0000] [.error] [src/action/start.go:func1:93] [103] [00:37:21.311] Error starting mongod : [00:37:21.311] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48711,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48711 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:37:20.182Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 15251 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:37:20.199+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:37:20.199+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:37:20.199+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48711,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:37:20.200+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:20.202+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:37:20.205+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:37:20.224+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:20.224+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:37:20.224+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:20.269+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:20.269+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:37:20.810+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863040:810528][15251:0x7f4a52c0dbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:37:20.911+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863040:911902][15251:0x7f4a52c0dbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:37:21.012+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863041:12424][15251:0x7f4a52c0dbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:37:21.127+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863041:127209][15251:0x7f4a52c0dbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:37:21.204+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863041:204504][15251:0x7f4a52c0dbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:37:21.260+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863041:260223][15251:0x7f4a52c0dbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:37:21.260+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863041:260295][15251:0x7f4a52c0dbc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:37:21.311+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:37:21.311] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48711 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:37:21.311] Error starting mongod : [00:37:21.311] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48711,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48711 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:37:20.182Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 15251 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:37:20.199+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:37:20.199+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:37:20.199+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48711,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:37:20.200+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:20.202+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:37:20.205+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:37:20.224+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:20.224+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:37:20.224+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:20.269+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:20.269+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:37:20.810+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863040:810528][15251:0x7f4a52c0dbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:37:20.911+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:37:21.311+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:37:21.311] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:37:21.311] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48711 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:37:21.311] Error starting mongod : [00:37:21.311] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48711,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48711 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:37:20.182Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 15251 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:37:20.199+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:37:20.199+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:37:20.199+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48711,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:37:20.200+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:20.202+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:37:20.205+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:37:20.224+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:20.224+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:37:20.224+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:20.269+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:20.269+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:37:20.810+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863040:810528][15251:0x7f4a52c0dbc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:37:21.311+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:37:21.311] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:37:21.311+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:37:21.311] Initiating stopEphemeralMongo [2021-10-22T00:37:21.311+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:37:21.311] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:37:21.311+0000] [.error] [src/director/director.go:executePlan:962] [00:37:21.311] Failed to apply action. Result = : [00:37:21.311] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:37:21.311] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48711 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:37:21.311] Error starting mongod : [00:37:21.311] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48711,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48711 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:37:20.182Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 15251 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:37:20.199+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:37:20.199+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:37:20.199+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48711,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:37:20.200+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:20.202+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:37:20.205+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:37:20.224+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:20.224+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:37:20.224+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:20.269+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:20.269+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:37:20.810+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:37:21.312+0000] [.error] [src/director/director.go:planAndExecute:567] [00:37:21.312] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:37:21.311] Failed to apply action. Result = : [00:37:21.311] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:37:21.311] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48711 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:37:21.311] Error starting mongod : [00:37:21.311] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48711,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48711 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:37:20.182Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 15251 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:37:20.199+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:37:20.199+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:37:20.199+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48711,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:37:20.200+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:20.202+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:37:20.205+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:37:20.224+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:20.224+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:37:20.224+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:20.269+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:20.269+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:37:21.322+0000] [.error] [src/director/director.go:mainLoop:399] [00:37:21.312] Failed to planAndExecute : [00:37:21.312] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:37:21.311] Failed to apply action. Result = : [00:37:21.311] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:37:21.311] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48711 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:37:21.311] Error starting mongod : [00:37:21.311] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48711,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48711 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:37:20.182Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 15251 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:37:20.199+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:37:20.199+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:37:20.199+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48711,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:37:20.200+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:20.202+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:37:20.205+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:37:20.224+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:20.224+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:37:20.224+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:20.269+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:20.269+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:37:21.848+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:37:21.848] clusterConfig unchanged [2021-10-22T00:37:22.068+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:22.068] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:23.069+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:23.069] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:24.070+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:24.070] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:25.071+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:25.071] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:26.072+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:26.072] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:27.073+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:27.073] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:27.555+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [00:37:27.555] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T00:37:27.559+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [00:37:27.559] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T00:37:27.559+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:27.559] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T00:37:27.559+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [00:37:27.559] Could not get version : [00:37:27.559] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T00:37:27.559+0000] [.warn] [filebeat/mspooler/slowqueries.go:PreFlushProcessing:141] [00:37:27.559] Error getting version for log lines. Skipping : [00:37:27.559] Could not get version : [00:37:27.559] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T00:37:28.073+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:28.073] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:29.073+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:29.073] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:29.984+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:37:29.984] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.172355092s [2021-10-22T00:37:30.072+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:30.072] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:31.072+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:31.072] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:31.951+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:37:31.951] clusterConfig unchanged [2021-10-22T00:37:32.072+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:32.072] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:33.073+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:33.072] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:34.075+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [00:37:34.075] lsblk cache invalidated, running lsblk command [2021-10-22T00:37:34.076+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:34.076] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:35.076+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:35.076] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:36.074+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:36.074] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:37.074+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:37.074] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:38.078+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:38.078] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:39.077+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:39.077] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:40.077+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:40.077] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:41.078+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:41.078] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:41.258+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:37:41.258] memory:rss=48967680 alloc=9302240 totalalloc=1354978456 mallocs=17911356 frees=17842216 heapinuse=13713408 heapobjects=69140 gcpausetotalns=1379019454 gcpauselastns=143206 gcnum=217 utimens=23433235000 stimens=6507139000 minflt=65534 majflt=404 [2021-10-22T00:37:42.072+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:37:42.072] clusterConfig unchanged [2021-10-22T00:37:42.078+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:42.078] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:42.488+0000] [.info] [src/director/director.go:computePlan:280] [00:37:42.488] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:37:42.488+0000] [.info] [src/director/director.go:tracef:794] [00:37:42.488] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:37:42.488+0000] [.info] [src/director/director.go:tracef:794] [00:37:42.488] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:37:42.488+0000] [.info] [src/director/director.go:tracef:794] [00:37:42.488] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:37:42.488+0000] [.info] [src/director/director.go:tracef:794] [00:37:42.488] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:37:42.489+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:37:42.489] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:37:42.636+0000] [.info] [src/director/director.go:tracef:794] [00:37:42.636] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:37:42.636+0000] [.info] [src/director/director.go:tracef:794] [00:37:42.636] because [2021-10-22T00:37:42.636+0000] [.info] [src/director/director.go:tracef:794] [00:37:42.636] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:37:42.636+0000] [.info] [src/director/director.go:tracef:794] [00:37:42.636] because ['currentState.CanBounceStop' = true] [2021-10-22T00:37:42.788+0000] [.info] [src/director/director.go:tracef:794] [00:37:42.788] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:37:42.788+0000] [.info] [src/director/director.go:tracef:794] [00:37:42.788] because [2021-10-22T00:37:42.788+0000] [.info] [src/director/director.go:tracef:794] [00:37:42.788] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:37:42.789+0000] [.info] [src/director/director.go:tracef:794] [00:37:42.789] because ['currentState.Up' = false] [2021-10-22T00:37:42.789+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:37:42.789] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:37:42.789+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:37:42.789] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:37:42.870+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:37:42.870] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:37:42.870+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:37:42.870] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:37:42.878+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:37:42.878] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:37:42.881+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:37:42.881] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:37:42.882+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:37:42.882] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:37:42.883+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:37:42.883] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:37:42.884+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:37:42.884] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:37:42.886+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:37:42.886] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:37:43.081+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:43.081] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:43.092+0000] [.info] [src/director/director.go:tracef:794] [00:37:43.092] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:37:43.092+0000] [.info] [src/director/director.go:tracef:794] [00:37:43.092] because ['currentState.Up' = false] [2021-10-22T00:37:43.293+0000] [.info] [src/director/director.go:tracef:794] [00:37:43.293] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:37:43.293+0000] [.info] [src/director/director.go:tracef:794] [00:37:43.293] because ['currentState.Up' = false] [2021-10-22T00:37:43.293+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:37:43.293] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:37:43.492+0000] [.info] [src/director/director.go:tracef:794] [00:37:43.492] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:37:43.492+0000] [.info] [src/director/director.go:tracef:794] [00:37:43.492] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:37:43.712+0000] [.info] [src/director/director.go:tracef:794] [00:37:43.712] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:37:43.712+0000] [.info] [src/director/director.go:tracef:794] [00:37:43.712] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:37:43.712+0000] [.info] [src/director/director.go:tracef:794] [00:37:43.712] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:37:43.712+0000] [.info] [src/director/director.go:tracef:794] [00:37:43.712] because ['currentState.Up' = false] [2021-10-22T00:37:43.712+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:37:43.712] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:37:43.712+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:37:43.712] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:37:43.745+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:37:43.745] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:37:43.746+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:37:43.746] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37456 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:37:44.091+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:44.091] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:44.486+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:37:44.486] ...Finished start [pid=34397] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37456,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:37:44.627+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:37:44.627] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 140.595896ms. [2021-10-22T00:37:44.627+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:37:44.627] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37456 (local=false) [2021-10-22T00:37:44.627+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:37:44.627] Dropping existing documents from system.replset [2021-10-22T00:37:44.828+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:37:44.828] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:37:44.831+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:37:44.831] Inserting oplogTruncateAfterPoint document [2021-10-22T00:37:44.831+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:37:44.831] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:37:44.832+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:37:44.832] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:37:44.832+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:37:44.832] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:37:44.845+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:37:44.845] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:37:44.845+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:37:44.845] Setting the minvalid document to [{_id ObjectID("617207d8523370002becc2ac")} {t -1} {ts {0 1}}] [2021-10-22T00:37:44.860+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:37:44.860] Inserted [{_id ObjectID("617207d8523370002becc2ac")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:37:44.860+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:37:44.860] Inserting new replica set configuration document. [2021-10-22T00:37:44.860+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:37:44.860] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:37:44.873+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:37:44.873] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:37:44.889+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:37:44.889] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:37:44.889+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:37:44.889] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:37:44.889+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:37:44.889] Initiating stopEphemeralMongo [2021-10-22T00:37:44.889+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:37:44.889] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37456 (local=false) [2021-10-22T00:37:44.889+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:37:44.889] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37456 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:37:45.105+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:45.105] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:45.272+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:37:45.272] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.137686ms. [2021-10-22T00:37:45.272+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:37:45.272] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37456 (local=false) [2021-10-22T00:37:45.272+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:37:45.272] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.218µs. [2021-10-22T00:37:45.273+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:37:45.273] Successfully finished running stopEphemeralMongo [2021-10-22T00:37:45.273+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:37:45.273] Successfully stopped existing ephemeral mongod. [2021-10-22T00:37:45.273+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:37:45.273] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:37:45.273+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:37:45.273] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:37:45.309+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:37:45.309] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:37:45.310+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:37:45.310] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48450 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:37:46.102+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:46.102] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:46.487+0000] [.error] [src/action/start.go:func1:93] [103] [00:37:46.487] Error starting mongod : [00:37:46.487] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48450 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:37:45.342Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 36035 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:37:45.373+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:37:45.373+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:37:45.373+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:37:45.375+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:45.377+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:37:45.377+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:37:45.401+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:45.401+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:37:45.412+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:45.460+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:45.460+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:37:45.983+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863065:983435][36035:0x7f495d115bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:37:46.086+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863066:86107][36035:0x7f495d115bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:37:46.186+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863066:186253][36035:0x7f495d115bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T00:37:46.299+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863066:299811][36035:0x7f495d115bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:37:46.373+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863066:373919][36035:0x7f495d115bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:37:46.436+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863066:436747][36035:0x7f495d115bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:37:46.436+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863066:436844][36035:0x7f495d115bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:37:46.487+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:37:46.487] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48450 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:37:46.487] Error starting mongod : [00:37:46.487] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48450 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:37:45.342Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 36035 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:37:45.373+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:37:45.373+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:37:45.373+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:37:45.375+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:45.377+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:37:45.377+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:37:45.401+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:45.401+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:37:45.412+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:45.460+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:45.460+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:37:45.983+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863065:983435][36035:0x7f495d115bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:37:46.086+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:37:46.487+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:37:46.487] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:37:46.487] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48450 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:37:46.487] Error starting mongod : [00:37:46.487] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48450 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:37:45.342Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 36035 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:37:45.373+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:37:45.373+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:37:45.373+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:37:45.375+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:45.377+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:37:45.377+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:37:45.401+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:45.401+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:37:45.412+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:45.460+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:45.460+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:37:45.983+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863065:983435][36035:0x7f495d115bc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:37:46.488+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:37:46.488] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:37:46.488+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:37:46.488] Initiating stopEphemeralMongo [2021-10-22T00:37:46.488+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:37:46.488] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:37:46.488+0000] [.error] [src/director/director.go:executePlan:962] [00:37:46.488] Failed to apply action. Result = : [00:37:46.487] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:37:46.487] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48450 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:37:46.487] Error starting mongod : [00:37:46.487] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48450 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:37:45.342Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 36035 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:37:45.373+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:37:45.373+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:37:45.373+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:37:45.375+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:45.377+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:37:45.377+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:37:45.401+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:45.401+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:37:45.412+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:45.460+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:45.460+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:37:45.983+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:37:46.488+0000] [.error] [src/director/director.go:planAndExecute:567] [00:37:46.488] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:37:46.488] Failed to apply action. Result = : [00:37:46.487] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:37:46.487] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48450 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:37:46.487] Error starting mongod : [00:37:46.487] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48450 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:37:45.342Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 36035 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:37:45.373+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:37:45.373+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:37:45.373+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:37:45.375+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:45.377+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:37:45.377+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:37:45.401+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:45.401+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:37:45.412+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:45.460+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:45.460+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:37:46.488+0000] [.error] [src/director/director.go:mainLoop:399] [00:37:46.488] Failed to planAndExecute : [00:37:46.488] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:37:46.488] Failed to apply action. Result = : [00:37:46.487] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:37:46.487] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48450 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:37:46.487] Error starting mongod : [00:37:46.487] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48450 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:37:45.342Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 36035 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:37:45.373+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:37:45.373+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:37:45.373+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:37:45.375+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:45.377+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:37:45.377+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:37:45.401+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:45.401+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:37:45.412+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:37:45.460+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:37:45.460+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:37:47.099+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:47.099] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:48.102+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:48.102] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:49.101+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:49.101] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:50.103+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:50.103] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:51.103+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:51.103] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:52.104+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:52.104] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:52.352+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:37:52.352] clusterConfig unchanged [2021-10-22T00:37:53.104+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:53.104] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:54.104+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:54.104] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:55.105+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:55.105] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:55.140+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:37:55.140] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.253990629s [2021-10-22T00:37:56.102+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:56.102] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:57.108+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:57.108] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:58.103+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:58.103] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:37:59.104+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:37:59.104] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:00.105+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:00.105] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:01.107+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:01.107] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:02.107+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:02.107] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:02.493+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:38:02.493] clusterConfig unchanged [2021-10-22T00:38:03.107+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:03.107] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:04.108+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:04.108] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:05.108+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:05.108] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:06.109+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:06.109] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:07.110+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:07.110] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:08.115+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:08.115] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:09.112+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:09.112] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:10.112+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:10.112] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:11.113+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:11.113] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:12.113+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:12.113] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:12.615+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:38:12.615] clusterConfig unchanged [2021-10-22T00:38:12.654+0000] [.info] [src/director/director.go:computePlan:280] [00:38:12.654] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:38:12.654+0000] [.info] [src/director/director.go:tracef:794] [00:38:12.654] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:38:12.654+0000] [.info] [src/director/director.go:tracef:794] [00:38:12.654] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:38:12.654+0000] [.info] [src/director/director.go:tracef:794] [00:38:12.654] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:38:12.654+0000] [.info] [src/director/director.go:tracef:794] [00:38:12.654] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:38:12.654+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:38:12.654] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:38:12.810+0000] [.info] [src/director/director.go:tracef:794] [00:38:12.810] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:38:12.810+0000] [.info] [src/director/director.go:tracef:794] [00:38:12.810] because [2021-10-22T00:38:12.810+0000] [.info] [src/director/director.go:tracef:794] [00:38:12.810] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:38:12.810+0000] [.info] [src/director/director.go:tracef:794] [00:38:12.810] because ['currentState.CanBounceStop' = true] [2021-10-22T00:38:12.976+0000] [.info] [src/director/director.go:tracef:794] [00:38:12.976] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:38:12.976+0000] [.info] [src/director/director.go:tracef:794] [00:38:12.976] because [2021-10-22T00:38:12.976+0000] [.info] [src/director/director.go:tracef:794] [00:38:12.976] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:38:12.976+0000] [.info] [src/director/director.go:tracef:794] [00:38:12.976] because ['currentState.Up' = false] [2021-10-22T00:38:12.976+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:38:12.976] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:38:12.976+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:38:12.976] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:38:13.058+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:38:13.058] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:38:13.066+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:38:13.066] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:38:13.067+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:38:13.067] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:38:13.068+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:38:13.068] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:38:13.069+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:38:13.069] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:38:13.069+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:38:13.069] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:38:13.069+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:38:13.069] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:38:13.070+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:38:13.070] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:38:13.135+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:13.135] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:13.330+0000] [.info] [src/director/director.go:tracef:794] [00:38:13.330] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:38:13.330+0000] [.info] [src/director/director.go:tracef:794] [00:38:13.330] because ['currentState.Up' = false] [2021-10-22T00:38:13.560+0000] [.info] [src/director/director.go:tracef:794] [00:38:13.560] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:38:13.560+0000] [.info] [src/director/director.go:tracef:794] [00:38:13.560] because ['currentState.Up' = false] [2021-10-22T00:38:13.561+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:38:13.561] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:38:13.750+0000] [.info] [src/director/director.go:tracef:794] [00:38:13.750] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:38:13.750+0000] [.info] [src/director/director.go:tracef:794] [00:38:13.750] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:38:13.959+0000] [.info] [src/director/director.go:tracef:794] [00:38:13.959] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:38:13.959+0000] [.info] [src/director/director.go:tracef:794] [00:38:13.959] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:38:13.960+0000] [.info] [src/director/director.go:tracef:794] [00:38:13.960] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:38:13.960+0000] [.info] [src/director/director.go:tracef:794] [00:38:13.960] because ['currentState.Up' = false] [2021-10-22T00:38:13.960+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:38:13.960] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:38:13.960+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:38:13.960] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:38:13.990+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:38:13.990] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:38:13.991+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:38:13.991] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58591 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:38:14.123+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:14.123] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:14.713+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:38:14.713] ...Finished start [pid=55190] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58591,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:38:14.848+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:38:14.848] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 135.06585ms. [2021-10-22T00:38:14.848+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:38:14.848] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:58591 (local=false) [2021-10-22T00:38:14.848+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:38:14.848] Dropping existing documents from system.replset [2021-10-22T00:38:15.012+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:38:15.012] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:38:15.016+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:38:15.016] Inserting oplogTruncateAfterPoint document [2021-10-22T00:38:15.016+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:38:15.016] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:38:15.017+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:38:15.017] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:38:15.017+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:38:15.017] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:38:15.032+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:38:15.032] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:38:15.032+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:38:15.032] Setting the minvalid document to [{_id ObjectID("617207f7523370002becc2ba")} {t -1} {ts {0 1}}] [2021-10-22T00:38:15.045+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:38:15.045] Inserted [{_id ObjectID("617207f7523370002becc2ba")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:38:15.045+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:38:15.045] Inserting new replica set configuration document. [2021-10-22T00:38:15.045+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:38:15.045] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:38:15.058+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:38:15.058] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:38:15.073+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:38:15.073] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:38:15.073+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:38:15.073] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:38:15.073+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:38:15.073] Initiating stopEphemeralMongo [2021-10-22T00:38:15.073+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:38:15.073] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:58591 (local=false) [2021-10-22T00:38:15.073+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:38:15.073] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:58591 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:38:15.123+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:15.123] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:15.479+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:38:15.479] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.750735ms. [2021-10-22T00:38:15.479+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:38:15.479] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:58591 (local=false) [2021-10-22T00:38:15.479+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:38:15.479] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.647µs. [2021-10-22T00:38:15.479+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:38:15.479] Successfully finished running stopEphemeralMongo [2021-10-22T00:38:15.479+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:38:15.479] Successfully stopped existing ephemeral mongod. [2021-10-22T00:38:15.479+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:38:15.479] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:38:15.479+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:38:15.479] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:38:15.515+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:38:15.515] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:38:15.516+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:38:15.516] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59026 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:38:16.127+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:16.127] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:16.670+0000] [.error] [src/action/start.go:func1:93] [103] [00:38:16.670] Error starting mongod : [00:38:16.670] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59026,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59026 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:38:15.549Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 56854 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:38:15.569+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:38:15.569+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:38:15.569+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59026,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:38:15.571+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:15.572+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:38:15.574+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:38:15.591+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:15.591+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:38:15.591+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:15.637+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:15.637+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:38:16.168+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863096:168586][56854:0x7fa34a460bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:38:16.270+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863096:270927][56854:0x7fa34a460bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:38:16.372+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863096:372358][56854:0x7fa34a460bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:38:16.484+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863096:484344][56854:0x7fa34a460bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:38:16.561+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863096:561327][56854:0x7fa34a460bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:38:16.619+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863096:619109][56854:0x7fa34a460bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:38:16.619+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863096:619181][56854:0x7fa34a460bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:38:16.670+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:38:16.670] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59026 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:38:16.670] Error starting mongod : [00:38:16.670] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59026,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59026 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:38:15.549Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 56854 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:38:15.569+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:38:15.569+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:38:15.569+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59026,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:38:15.571+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:15.572+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:38:15.574+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:38:15.591+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:15.591+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:38:15.591+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:15.637+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:15.637+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:38:16.168+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863096:168586][56854:0x7fa34a460bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:38:16.270+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:38:16.671+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:38:16.671] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:38:16.670] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59026 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:38:16.670] Error starting mongod : [00:38:16.670] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59026,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59026 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:38:15.549Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 56854 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:38:15.569+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:38:15.569+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:38:15.569+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59026,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:38:15.571+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:15.572+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:38:15.574+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:38:15.591+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:15.591+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:38:15.591+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:15.637+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:15.637+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:38:16.168+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863096:168586][56854:0x7fa34a460bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:38:16.671+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:38:16.671] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:38:16.671+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:38:16.671] Initiating stopEphemeralMongo [2021-10-22T00:38:16.671+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:38:16.671] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:38:16.671+0000] [.error] [src/director/director.go:executePlan:962] [00:38:16.671] Failed to apply action. Result = : [00:38:16.671] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:38:16.670] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59026 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:38:16.670] Error starting mongod : [00:38:16.670] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59026,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59026 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:38:15.549Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 56854 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:38:15.569+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:38:15.569+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:38:15.569+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59026,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:38:15.571+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:15.572+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:38:15.574+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:38:15.591+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:15.591+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:38:15.591+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:15.637+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:15.637+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:38:16.168+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:38:16.671+0000] [.error] [src/director/director.go:planAndExecute:567] [00:38:16.671] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:38:16.671] Failed to apply action. Result = : [00:38:16.671] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:38:16.670] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59026 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:38:16.670] Error starting mongod : [00:38:16.670] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59026,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59026 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:38:15.549Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 56854 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:38:15.569+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:38:15.569+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:38:15.569+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59026,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:38:15.571+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:15.572+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:38:15.574+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:38:15.591+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:15.591+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:38:15.591+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:15.637+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:15.637+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:38:16.672+0000] [.error] [src/director/director.go:mainLoop:399] [00:38:16.672] Failed to planAndExecute : [00:38:16.671] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:38:16.671] Failed to apply action. Result = : [00:38:16.671] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:38:16.670] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59026 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:38:16.670] Error starting mongod : [00:38:16.670] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59026,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59026 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:38:15.549Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 56854 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:38:15.569+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:38:15.569+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:38:15.569+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59026,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:38:15.571+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:15.572+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:38:15.574+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:38:15.591+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:15.591+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:38:15.591+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:15.637+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:15.637+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:38:17.124+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:17.124] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:18.126+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:18.126] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:19.127+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:19.127] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:20.127+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:20.127] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:21.127+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:21.127] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:22.125+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:22.125] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:22.860+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:38:22.860] clusterConfig unchanged [2021-10-22T00:38:23.129+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:23.129] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:24.129+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:24.129] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:25.129+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:25.129] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:25.257+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:38:25.257] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.187231193s [2021-10-22T00:38:26.127+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:26.127] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:27.133+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:27.133] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:28.128+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:28.128] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:29.128+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:29.128] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:30.129+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:30.129] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:31.130+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:31.130] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:32.130+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:32.130] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:32.996+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:38:32.996] clusterConfig unchanged [2021-10-22T00:38:33.131+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:33.131] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:34.131+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:34.131] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:35.132+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:35.132] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:36.133+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:36.132] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:37.133+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:37.133] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:38.138+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:38.138] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:39.134+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:39.134] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:40.142+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:40.142] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:41.135+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:41.135] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:41.258+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:38:41.258] memory:rss=48660480 alloc=7842032 totalalloc=1491486136 mallocs=19673693 frees=19620367 heapinuse=13074432 heapobjects=53326 gcpausetotalns=1381128974 gcpauselastns=64096 gcnum=240 utimens=25764475000 stimens=7208163000 minflt=71800 majflt=404 [2021-10-22T00:38:42.135+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:42.135] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:42.830+0000] [.info] [src/director/director.go:computePlan:280] [00:38:42.830] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:38:42.830+0000] [.info] [src/director/director.go:tracef:794] [00:38:42.830] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:38:42.830+0000] [.info] [src/director/director.go:tracef:794] [00:38:42.830] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:38:42.830+0000] [.info] [src/director/director.go:tracef:794] [00:38:42.830] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:38:42.831+0000] [.info] [src/director/director.go:tracef:794] [00:38:42.830] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:38:42.831+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:38:42.831] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:38:42.984+0000] [.info] [src/director/director.go:tracef:794] [00:38:42.984] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:38:42.984+0000] [.info] [src/director/director.go:tracef:794] [00:38:42.984] because [2021-10-22T00:38:42.984+0000] [.info] [src/director/director.go:tracef:794] [00:38:42.984] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:38:42.984+0000] [.info] [src/director/director.go:tracef:794] [00:38:42.984] because ['currentState.CanBounceStop' = true] [2021-10-22T00:38:43.134+0000] [.info] [src/director/director.go:tracef:794] [00:38:43.134] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:38:43.134+0000] [.info] [src/director/director.go:tracef:794] [00:38:43.134] because [2021-10-22T00:38:43.138+0000] [.info] [src/director/director.go:tracef:794] [00:38:43.137] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:38:43.139+0000] [.info] [src/director/director.go:tracef:794] [00:38:43.139] because ['currentState.Up' = false] [2021-10-22T00:38:43.140+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:38:43.139] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:38:43.140+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:38:43.140] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:38:43.143+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:43.143] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:43.206+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:38:43.206] clusterConfig unchanged [2021-10-22T00:38:43.227+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:38:43.227] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:38:43.235+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:38:43.235] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:38:43.235+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:38:43.235] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:38:43.236+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:38:43.236] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:38:43.237+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:38:43.237] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:38:43.237+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:38:43.237] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:38:43.237+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:38:43.237] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:38:43.239+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:38:43.239] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:38:43.447+0000] [.info] [src/director/director.go:tracef:794] [00:38:43.447] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:38:43.447+0000] [.info] [src/director/director.go:tracef:794] [00:38:43.447] because ['currentState.Up' = false] [2021-10-22T00:38:43.674+0000] [.info] [src/director/director.go:tracef:794] [00:38:43.674] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:38:43.674+0000] [.info] [src/director/director.go:tracef:794] [00:38:43.674] because ['currentState.Up' = false] [2021-10-22T00:38:43.674+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:38:43.674] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:38:43.879+0000] [.info] [src/director/director.go:tracef:794] [00:38:43.879] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:38:43.879+0000] [.info] [src/director/director.go:tracef:794] [00:38:43.879] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:38:44.063+0000] [.info] [src/director/director.go:tracef:794] [00:38:44.063] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:38:44.063+0000] [.info] [src/director/director.go:tracef:794] [00:38:44.063] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:38:44.063+0000] [.info] [src/director/director.go:tracef:794] [00:38:44.063] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:38:44.063+0000] [.info] [src/director/director.go:tracef:794] [00:38:44.063] because ['currentState.Up' = false] [2021-10-22T00:38:44.063+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:38:44.063] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:38:44.063+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:38:44.063] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:38:44.098+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:38:44.098] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:38:44.098+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:38:44.098] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36964 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:38:44.148+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:44.148] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:44.310+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:38:44.310] clusterConfig unchanged [2021-10-22T00:38:44.812+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:38:44.812] ...Finished start [pid=1856] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36964,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:38:44.923+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:38:44.923] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 110.869486ms. [2021-10-22T00:38:44.923+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:38:44.923] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36964 (local=false) [2021-10-22T00:38:44.923+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:38:44.923] Dropping existing documents from system.replset [2021-10-22T00:38:45.065+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:38:45.065] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:38:45.068+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:38:45.068] Inserting oplogTruncateAfterPoint document [2021-10-22T00:38:45.068+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:38:45.068] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:38:45.069+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:38:45.069] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:38:45.069+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:38:45.069] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:38:45.082+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:38:45.082] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:38:45.082+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:38:45.082] Setting the minvalid document to [{_id ObjectID("61720815523370002becc2c7")} {t -1} {ts {0 1}}] [2021-10-22T00:38:45.096+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:38:45.096] Inserted [{_id ObjectID("61720815523370002becc2c7")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:38:45.096+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:38:45.096] Inserting new replica set configuration document. [2021-10-22T00:38:45.096+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:38:45.096] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:38:45.109+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:38:45.109] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:38:45.113+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:38:45.113] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:38:45.113+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:38:45.113] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:38:45.113+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:38:45.113] Initiating stopEphemeralMongo [2021-10-22T00:38:45.113+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:38:45.113] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36964 (local=false) [2021-10-22T00:38:45.113+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:38:45.113] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36964 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:38:45.156+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:45.156] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:45.442+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:38:45.442] clusterConfig unchanged [2021-10-22T00:38:45.479+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:38:45.479] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.768773ms. [2021-10-22T00:38:45.479+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:38:45.479] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36964 (local=false) [2021-10-22T00:38:45.479+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:38:45.479] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.07µs. [2021-10-22T00:38:45.479+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:38:45.479] Successfully finished running stopEphemeralMongo [2021-10-22T00:38:45.479+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:38:45.479] Successfully stopped existing ephemeral mongod. [2021-10-22T00:38:45.479+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:38:45.479] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:38:45.479+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:38:45.479] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:38:45.515+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:38:45.515] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:38:45.515+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:38:45.515] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51599 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:38:46.158+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:46.158] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:46.555+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:38:46.555] clusterConfig unchanged [2021-10-22T00:38:46.672+0000] [.error] [src/action/start.go:func1:93] [103] [00:38:46.672] Error starting mongod : [00:38:46.672] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51599 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:38:45.548Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 3414 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:38:45.577+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:38:45.577+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:38:45.577+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:38:45.579+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:45.582+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:38:45.599+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:38:45.632+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:45.632+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:38:45.632+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:45.679+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:45.679+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:38:46.174+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863126:174352][3414:0x7f578956dbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:38:46.278+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863126:278794][3414:0x7f578956dbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:38:46.380+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863126:380121][3414:0x7f578956dbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:38:46.490+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863126:490282][3414:0x7f578956dbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:38:46.564+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863126:564680][3414:0x7f578956dbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:38:46.621+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863126:621052][3414:0x7f578956dbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:38:46.621+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863126:621121][3414:0x7f578956dbc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:38:46.673+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:38:46.672] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51599 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:38:46.672] Error starting mongod : [00:38:46.672] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51599 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:38:45.548Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 3414 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:38:45.577+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:38:45.577+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:38:45.577+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:38:45.579+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:45.582+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:38:45.599+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:38:45.632+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:45.632+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:38:45.632+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:45.679+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:45.679+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:38:46.174+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863126:174352][3414:0x7f578956dbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:38:46.278+00:00"},"s...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:38:46.673+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:38:46.673] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:38:46.672] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51599 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:38:46.672] Error starting mongod : [00:38:46.672] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51599 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:38:45.548Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 3414 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:38:45.577+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:38:45.577+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:38:45.577+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:38:45.579+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:45.582+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:38:45.599+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:38:45.632+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:45.632+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:38:45.632+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:45.679+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:45.679+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:38:46.174+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863126:174352][3414:0x7f578956dbc0... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:38:46.673+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:38:46.673] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:38:46.673+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:38:46.673] Initiating stopEphemeralMongo [2021-10-22T00:38:46.673+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:38:46.673] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:38:46.673+0000] [.error] [src/director/director.go:executePlan:962] [00:38:46.673] Failed to apply action. Result = : [00:38:46.673] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:38:46.672] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51599 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:38:46.672] Error starting mongod : [00:38:46.672] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51599 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:38:45.548Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 3414 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:38:45.577+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:38:45.577+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:38:45.577+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:38:45.579+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:45.582+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:38:45.599+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:38:45.632+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:45.632+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:38:45.632+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:45.679+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:45.679+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:38:46.174+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:38:46.674+0000] [.error] [src/director/director.go:planAndExecute:567] [00:38:46.674] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:38:46.673] Failed to apply action. Result = : [00:38:46.673] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:38:46.672] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51599 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:38:46.672] Error starting mongod : [00:38:46.672] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51599 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:38:45.548Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 3414 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:38:45.577+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:38:45.577+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:38:45.577+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:38:45.579+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:45.582+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:38:45.599+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:38:45.632+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:45.632+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:38:45.632+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:45.679+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:45.679+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:38:46.674+0000] [.error] [src/director/director.go:mainLoop:399] [00:38:46.674] Failed to planAndExecute : [00:38:46.674] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:38:46.673] Failed to apply action. Result = : [00:38:46.673] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:38:46.672] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51599 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:38:46.672] Error starting mongod : [00:38:46.672] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51599 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:38:45.548Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 3414 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:38:45.577+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:38:45.577+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:38:45.577+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51599,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:38:45.579+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:45.582+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:38:45.599+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:38:45.632+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:45.632+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:38:45.632+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:38:45.679+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:38:45.679+00:00"},"s":"I", "c":"NETWORK", "id"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:38:47.156+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:47.156] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:47.805+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:38:47.805] clusterConfig unchanged [2021-10-22T00:38:48.158+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:48.158] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:49.158+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:49.158] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:50.156+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:50.156] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:51.158+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:51.158] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:52.163+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:52.163] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:53.159+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:53.159] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:54.159+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:54.159] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:55.160+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:55.160] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:55.263+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:38:55.263] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.024505063s [2021-10-22T00:38:56.157+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:56.157] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:57.160+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:57.160] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:57.927+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:38:57.927] clusterConfig unchanged [2021-10-22T00:38:58.158+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:58.158] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:38:59.158+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:38:59.158] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:00.159+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:00.159] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:01.160+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:01.160] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:02.161+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:02.161] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:03.161+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:03.161] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:04.167+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:04.167] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:05.163+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:05.163] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:06.163+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:06.163] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:07.164+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:07.164] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:08.049+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:39:08.049] clusterConfig unchanged [2021-10-22T00:39:08.165+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:08.165] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:09.165+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:09.165] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:10.165+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:10.165] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:11.166+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:11.166] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:12.166+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:12.166] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:13.167+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:13.167] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:14.169+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:14.169] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:15.173+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:15.173] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:16.169+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:16.169] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:17.170+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:17.170] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:17.837+0000] [.info] [src/director/director.go:computePlan:280] [00:39:17.837] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:39:17.837+0000] [.info] [src/director/director.go:tracef:794] [00:39:17.837] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:39:17.837+0000] [.info] [src/director/director.go:tracef:794] [00:39:17.837] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:39:17.837+0000] [.info] [src/director/director.go:tracef:794] [00:39:17.837] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:39:17.837+0000] [.info] [src/director/director.go:tracef:794] [00:39:17.837] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:39:17.837+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:39:17.837] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:39:17.990+0000] [.info] [src/director/director.go:tracef:794] [00:39:17.990] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:39:17.990+0000] [.info] [src/director/director.go:tracef:794] [00:39:17.990] because [2021-10-22T00:39:17.990+0000] [.info] [src/director/director.go:tracef:794] [00:39:17.990] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:39:17.990+0000] [.info] [src/director/director.go:tracef:794] [00:39:17.990] because ['currentState.CanBounceStop' = true] [2021-10-22T00:39:18.174+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:18.174] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:18.190+0000] [.info] [src/director/director.go:tracef:794] [00:39:18.190] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:39:18.190+0000] [.info] [src/director/director.go:tracef:794] [00:39:18.190] because [2021-10-22T00:39:18.191+0000] [.info] [src/director/director.go:tracef:794] [00:39:18.191] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:39:18.191+0000] [.info] [src/director/director.go:tracef:794] [00:39:18.191] because ['currentState.Up' = false] [2021-10-22T00:39:18.191+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:39:18.191] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:39:18.191+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:39:18.191] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:39:18.259+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:39:18.259] clusterConfig unchanged [2021-10-22T00:39:18.268+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:39:18.268] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:39:18.276+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:39:18.276] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:39:18.277+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:39:18.277] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:39:18.277+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:39:18.277] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:39:18.278+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:39:18.278] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:39:18.279+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:39:18.279] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:39:18.279+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:39:18.279] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:39:18.280+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:39:18.280] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:39:18.522+0000] [.info] [src/director/director.go:tracef:794] [00:39:18.522] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:39:18.523+0000] [.info] [src/director/director.go:tracef:794] [00:39:18.523] because ['currentState.Up' = false] [2021-10-22T00:39:18.734+0000] [.info] [src/director/director.go:tracef:794] [00:39:18.734] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:39:18.734+0000] [.info] [src/director/director.go:tracef:794] [00:39:18.734] because ['currentState.Up' = false] [2021-10-22T00:39:18.734+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:39:18.734] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:39:18.950+0000] [.info] [src/director/director.go:tracef:794] [00:39:18.950] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:39:18.951+0000] [.info] [src/director/director.go:tracef:794] [00:39:18.950] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:39:19.165+0000] [.info] [src/director/director.go:tracef:794] [00:39:19.165] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:39:19.165+0000] [.info] [src/director/director.go:tracef:794] [00:39:19.165] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:39:19.166+0000] [.info] [src/director/director.go:tracef:794] [00:39:19.166] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:39:19.166+0000] [.info] [src/director/director.go:tracef:794] [00:39:19.166] because ['currentState.Up' = false] [2021-10-22T00:39:19.166+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:39:19.166] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:39:19.166+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:39:19.166] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:39:19.176+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:19.176] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:19.199+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:39:19.199] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:39:19.200+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:39:19.200] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40540 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:39:19.367+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:39:19.367] clusterConfig unchanged [2021-10-22T00:39:19.925+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:39:19.925] ...Finished start [pid=22730] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40540,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:39:20.051+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:39:20.050] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 124.967712ms. [2021-10-22T00:39:20.051+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:39:20.051] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40540 (local=false) [2021-10-22T00:39:20.051+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:39:20.051] Dropping existing documents from system.replset [2021-10-22T00:39:20.177+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:20.177] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:20.179+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:39:20.179] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:39:20.182+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:39:20.182] Inserting oplogTruncateAfterPoint document [2021-10-22T00:39:20.182+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:39:20.182] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:39:20.183+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:39:20.183] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:39:20.183+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:39:20.183] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:39:20.196+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:39:20.196] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:39:20.196+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:39:20.196] Setting the minvalid document to [{_id ObjectID("61720838523370002becc2d5")} {t -1} {ts {0 1}}] [2021-10-22T00:39:20.211+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:39:20.210] Inserted [{_id ObjectID("61720838523370002becc2d5")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:39:20.211+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:39:20.211] Inserting new replica set configuration document. [2021-10-22T00:39:20.211+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:39:20.211] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:39:20.234+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:39:20.234] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:39:20.259+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:39:20.259] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:39:20.259+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:39:20.259] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:39:20.259+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:39:20.259] Initiating stopEphemeralMongo [2021-10-22T00:39:20.259+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:39:20.259] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40540 (local=false) [2021-10-22T00:39:20.259+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:39:20.259] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40540 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:39:20.474+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:39:20.474] clusterConfig unchanged [2021-10-22T00:39:20.752+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:39:20.752] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.186615ms. [2021-10-22T00:39:20.752+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:39:20.752] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40540 (local=false) [2021-10-22T00:39:20.752+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:39:20.752] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 38.093µs. [2021-10-22T00:39:20.752+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:39:20.752] Successfully finished running stopEphemeralMongo [2021-10-22T00:39:20.752+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:39:20.752] Successfully stopped existing ephemeral mongod. [2021-10-22T00:39:20.752+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:39:20.752] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:39:20.752+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:39:20.752] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:39:20.789+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:39:20.789] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:39:20.789+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:39:20.789] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51608 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:39:21.180+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:21.180] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:21.577+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:39:21.577] clusterConfig unchanged [2021-10-22T00:39:21.947+0000] [.error] [src/action/start.go:func1:93] [103] [00:39:21.947] Error starting mongod : [00:39:21.947] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51608,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51608 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 24355 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:39:20.842+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:39:20.842+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:39:20.842+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51608,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:39:20.844+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:20.847+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:39:20.858+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:39:20.898+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:20.898+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:20.899+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:20.944+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:20.944+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:21.441+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863161:441144][24355:0x7fd4af6cebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:39:21.543+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863161:543778][24355:0x7fd4af6cebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:39:21.644+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863161:644428][24355:0x7fd4af6cebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:39:21.756+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863161:755990][24355:0x7fd4af6cebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:39:21.834+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863161:834179][24355:0x7fd4af6cebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:39:21.895+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863161:895852][24355:0x7fd4af6cebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:39:21.895+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863161:895927][24355:0x7fd4af6cebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:39:21.897+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:39:21.948+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:39:21.948] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51608 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:39:21.947] Error starting mongod : [00:39:21.947] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51608,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51608 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 24355 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:39:20.842+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:39:20.842+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:39:20.842+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51608,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:39:20.844+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:20.847+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:39:20.858+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:39:20.898+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:20.898+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:20.899+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:20.944+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:20.944+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:21.441+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863161:441144][24355:0x7fd4af6cebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:39:21.543+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863161:543778][24355:0x7fd4af6cebc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:39:21.948+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:39:21.948] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:39:21.948] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51608 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:39:21.947] Error starting mongod : [00:39:21.947] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51608,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51608 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 24355 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:39:20.842+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:39:20.842+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:39:20.842+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51608,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:39:20.844+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:20.847+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:39:20.858+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:39:20.898+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:20.898+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:20.899+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:20.944+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:20.944+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:21.441+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863161:441144][24355:0x7fd4af6cebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:39:21.543+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:39:21.948+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:39:21.948] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:39:21.948+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:39:21.948] Initiating stopEphemeralMongo [2021-10-22T00:39:21.948+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:39:21.948] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:39:21.951+0000] [.error] [src/director/director.go:executePlan:962] [00:39:21.951] Failed to apply action. Result = : [00:39:21.948] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:39:21.948] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51608 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:39:21.947] Error starting mongod : [00:39:21.947] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51608,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51608 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 24355 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:39:20.842+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:39:20.842+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:39:20.842+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51608,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:39:20.844+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:20.847+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:39:20.858+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:39:20.898+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:20.898+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:20.899+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:20.944+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:20.944+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:21.441+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863161:441144][24355:0x7fd4af6cebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:39:21.951+0000] [.error] [src/director/director.go:planAndExecute:567] [00:39:21.951] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:39:21.951] Failed to apply action. Result = : [00:39:21.948] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:39:21.948] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51608 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:39:21.947] Error starting mongod : [00:39:21.947] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51608,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51608 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 24355 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:39:20.842+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:39:20.842+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:39:20.842+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51608,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:39:20.844+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:20.847+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:39:20.858+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:39:20.898+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:20.898+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:20.899+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:20.944+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:20.944+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:21.441+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:39:21.952+0000] [.error] [src/director/director.go:mainLoop:399] [00:39:21.952] Failed to planAndExecute : [00:39:21.951] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:39:21.951] Failed to apply action. Result = : [00:39:21.948] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:39:21.948] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51608 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:39:21.947] Error starting mongod : [00:39:21.947] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51608,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51608 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 24355 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:39:20.842+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:39:20.842+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:39:20.842+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51608,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:39:20.844+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:20.847+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:39:20.858+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:39:20.898+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:20.898+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:20.899+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:20.944+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:20.944+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:21.441+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:39:22.179+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:22.179] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:22.807+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:39:22.807] clusterConfig unchanged [2021-10-22T00:39:23.180+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:23.180] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:24.181+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:24.181] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:25.183+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:25.183] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:26.183+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:26.183] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:27.184+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:27.184] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:28.185+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:28.185] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:29.191+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:29.191] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:30.187+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:30.187] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:30.546+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:39:30.546] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.266052249s [2021-10-22T00:39:31.184+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:31.184] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:32.185+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:32.185] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:32.909+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:39:32.909] clusterConfig unchanged [2021-10-22T00:39:33.186+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:33.186] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:34.186+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:34.186] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:35.186+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:35.186] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:36.187+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:36.187] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:37.187+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:37.187] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:38.188+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:38.188] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:39.188+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:39.188] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:40.188+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:40.188] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:41.188+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:41.188] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:41.259+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:39:41.259] memory:rss=50528256 alloc=6452424 totalalloc=1627766432 mallocs=21432755 frees=21394734 heapinuse=12877824 heapobjects=38021 gcpausetotalns=1383300221 gcpauselastns=65723 gcnum=262 utimens=27988820000 stimens=7831535000 minflt=77861 majflt=405 [2021-10-22T00:39:42.189+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:42.189] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:43.032+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:39:43.032] clusterConfig unchanged [2021-10-22T00:39:43.189+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:43.189] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:44.191+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:44.191] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:45.191+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:45.191] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:46.192+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:46.192] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:47.192+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:47.192] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:48.196+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:48.196] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:49.193+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:49.193] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:50.194+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:50.194] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:51.194+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:51.194] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:52.195+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:52.195] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:53.116+0000] [.info] [src/director/director.go:computePlan:280] [00:39:53.116] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:39:53.116+0000] [.info] [src/director/director.go:tracef:794] [00:39:53.116] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:39:53.116+0000] [.info] [src/director/director.go:tracef:794] [00:39:53.116] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:39:53.116+0000] [.info] [src/director/director.go:tracef:794] [00:39:53.116] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:39:53.116+0000] [.info] [src/director/director.go:tracef:794] [00:39:53.116] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:39:53.116+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:39:53.116] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:39:53.134+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:39:53.134] clusterConfig unchanged [2021-10-22T00:39:53.197+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:53.197] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:53.271+0000] [.info] [src/director/director.go:tracef:794] [00:39:53.271] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:39:53.271+0000] [.info] [src/director/director.go:tracef:794] [00:39:53.271] because [2021-10-22T00:39:53.271+0000] [.info] [src/director/director.go:tracef:794] [00:39:53.271] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:39:53.271+0000] [.info] [src/director/director.go:tracef:794] [00:39:53.271] because ['currentState.CanBounceStop' = true] [2021-10-22T00:39:53.424+0000] [.info] [src/director/director.go:tracef:794] [00:39:53.424] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:39:53.424+0000] [.info] [src/director/director.go:tracef:794] [00:39:53.424] because [2021-10-22T00:39:53.424+0000] [.info] [src/director/director.go:tracef:794] [00:39:53.424] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:39:53.424+0000] [.info] [src/director/director.go:tracef:794] [00:39:53.424] because ['currentState.Up' = false] [2021-10-22T00:39:53.424+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:39:53.424] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:39:53.424+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:39:53.424] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:39:53.508+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:39:53.508] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:39:53.516+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:39:53.516] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:39:53.517+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:39:53.517] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:39:53.518+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:39:53.518] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:39:53.519+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:39:53.519] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:39:53.519+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:39:53.519] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:39:53.519+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:39:53.519] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:39:53.520+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:39:53.520] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:39:53.729+0000] [.info] [src/director/director.go:tracef:794] [00:39:53.729] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:39:53.729+0000] [.info] [src/director/director.go:tracef:794] [00:39:53.729] because ['currentState.Up' = false] [2021-10-22T00:39:53.939+0000] [.info] [src/director/director.go:tracef:794] [00:39:53.939] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:39:53.939+0000] [.info] [src/director/director.go:tracef:794] [00:39:53.939] because ['currentState.Up' = false] [2021-10-22T00:39:53.939+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:39:53.939] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:39:54.151+0000] [.info] [src/director/director.go:tracef:794] [00:39:54.151] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:39:54.151+0000] [.info] [src/director/director.go:tracef:794] [00:39:54.151] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:39:54.197+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:54.197] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:54.356+0000] [.info] [src/director/director.go:tracef:794] [00:39:54.356] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:39:54.356+0000] [.info] [src/director/director.go:tracef:794] [00:39:54.356] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:39:54.357+0000] [.info] [src/director/director.go:tracef:794] [00:39:54.357] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:39:54.357+0000] [.info] [src/director/director.go:tracef:794] [00:39:54.357] because ['currentState.Up' = false] [2021-10-22T00:39:54.357+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:39:54.357] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:39:54.357+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:39:54.357] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:39:54.387+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:39:54.387] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:39:54.388+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:39:54.388] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59612 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:39:55.101+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:39:55.101] ...Finished start [pid=43377] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59612,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:39:55.212+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:55.212] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:55.241+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:39:55.241] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 140.141572ms. [2021-10-22T00:39:55.241+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:39:55.241] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:59612 (local=false) [2021-10-22T00:39:55.241+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:39:55.241] Dropping existing documents from system.replset [2021-10-22T00:39:55.374+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:39:55.374] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:39:55.378+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:39:55.378] Inserting oplogTruncateAfterPoint document [2021-10-22T00:39:55.378+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:39:55.378] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:39:55.379+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:39:55.379] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:39:55.379+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:39:55.379] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:39:55.391+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:39:55.391] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:39:55.391+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:39:55.391] Setting the minvalid document to [{_id ObjectID("6172085b523370002becc2e3")} {t -1} {ts {0 1}}] [2021-10-22T00:39:55.404+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:39:55.404] Inserted [{_id ObjectID("6172085b523370002becc2e3")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:39:55.404+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:39:55.404] Inserting new replica set configuration document. [2021-10-22T00:39:55.404+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:39:55.404] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:39:55.416+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:39:55.416] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:39:55.421+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:39:55.421] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:39:55.421+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:39:55.421] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:39:55.421+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:39:55.421] Initiating stopEphemeralMongo [2021-10-22T00:39:55.421+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:39:55.421] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:59612 (local=false) [2021-10-22T00:39:55.421+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:39:55.421] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:59612 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:39:55.788+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:39:55.788] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.113035ms. [2021-10-22T00:39:55.788+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:39:55.788] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:59612 (local=false) [2021-10-22T00:39:55.788+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:39:55.788] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 38.851µs. [2021-10-22T00:39:55.788+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:39:55.788] Successfully finished running stopEphemeralMongo [2021-10-22T00:39:55.788+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:39:55.788] Successfully stopped existing ephemeral mongod. [2021-10-22T00:39:55.788+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:39:55.788] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:39:55.788+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:39:55.788] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:39:55.824+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:39:55.824] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:39:55.827+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:39:55.827] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 44794 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:39:56.208+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:56.208] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:56.983+0000] [.error] [src/action/start.go:func1:93] [103] [00:39:56.983] Error starting mongod : [00:39:56.983] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44794,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 44794 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:39:55.858Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 45021 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:39:55.877+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:39:55.877+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:39:55.877+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44794,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:39:55.879+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:55.881+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:39:55.881+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:39:55.921+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:55.921+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:55.932+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:55.977+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:55.977+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:56.491+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863196:491259][45021:0x7fe5a962bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:39:56.594+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863196:594805][45021:0x7fe5a962bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:39:56.693+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863196:693599][45021:0x7fe5a962bbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:39:56.803+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863196:803790][45021:0x7fe5a962bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:39:56.875+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863196:875160][45021:0x7fe5a962bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:39:56.934+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863196:934156][45021:0x7fe5a962bbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:39:56.934+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863196:934232][45021:0x7fe5a962bbc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:39:56.983+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:39:56.983] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:44794 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:39:56.983] Error starting mongod : [00:39:56.983] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44794,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 44794 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:39:55.858Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 45021 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:39:55.877+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:39:55.877+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:39:55.877+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44794,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:39:55.879+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:55.881+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:39:55.881+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:39:55.921+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:55.921+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:55.932+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:55.977+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:55.977+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:56.491+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863196:491259][45021:0x7fe5a962bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:39:56.594+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:39:56.983+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:39:56.983] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:39:56.983] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:44794 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:39:56.983] Error starting mongod : [00:39:56.983] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44794,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 44794 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:39:55.858Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 45021 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:39:55.877+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:39:55.877+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:39:55.877+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44794,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:39:55.879+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:55.881+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:39:55.881+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:39:55.921+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:55.921+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:55.932+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:55.977+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:55.977+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:56.491+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863196:491259][45021:0x7fe5a962bbc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:39:56.983+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:39:56.983] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:39:56.983+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:39:56.983] Initiating stopEphemeralMongo [2021-10-22T00:39:56.983+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:39:56.983] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:39:56.984+0000] [.error] [src/director/director.go:executePlan:962] [00:39:56.984] Failed to apply action. Result = : [00:39:56.983] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:39:56.983] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:44794 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:39:56.983] Error starting mongod : [00:39:56.983] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44794,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 44794 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:39:55.858Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 45021 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:39:55.877+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:39:55.877+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:39:55.877+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44794,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:39:55.879+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:55.881+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:39:55.881+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:39:55.921+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:55.921+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:55.932+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:55.977+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:55.977+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:56.491+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:39:56.984+0000] [.error] [src/director/director.go:planAndExecute:567] [00:39:56.984] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:39:56.984] Failed to apply action. Result = : [00:39:56.983] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:39:56.983] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:44794 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:39:56.983] Error starting mongod : [00:39:56.983] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44794,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 44794 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:39:55.858Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 45021 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:39:55.877+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:39:55.877+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:39:55.877+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44794,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:39:55.879+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:55.881+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:39:55.881+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:39:55.921+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:55.921+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:55.932+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:55.977+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:55.977+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:39:56.984+0000] [.error] [src/director/director.go:mainLoop:399] [00:39:56.984] Failed to planAndExecute : [00:39:56.984] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:39:56.984] Failed to apply action. Result = : [00:39:56.983] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:39:56.983] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:44794 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:39:56.983] Error starting mongod : [00:39:56.983] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44794,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 44794 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:39:55.858Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 45021 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:39:55.877+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:39:55.877+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:39:55.877+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44794,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:39:55.879+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:55.881+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:39:55.881+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:39:55.921+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:55.921+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:39:55.932+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:39:55.977+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:39:55.977+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:39:57.207+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:57.207] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:58.207+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:58.207] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:39:59.209+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:39:59.209] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:00.210+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:00.210] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:01.215+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:01.215] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:02.213+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:02.213] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:03.211+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:03.211] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:03.404+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:40:03.404] clusterConfig unchanged [2021-10-22T00:40:04.213+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:04.213] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:05.212+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:05.212] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:05.685+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:40:05.685] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.164812054s [2021-10-22T00:40:06.212+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:06.212] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:07.213+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:07.213] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:08.214+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:08.214] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:09.214+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:09.214] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:10.221+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:10.221] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:11.217+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:11.217] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:12.217+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:12.217] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:13.218+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:13.218] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:13.526+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:40:13.526] clusterConfig unchanged [2021-10-22T00:40:14.219+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:14.219] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:15.219+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:15.219] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:16.219+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:16.219] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:17.220+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:17.220] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:18.221+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:18.221] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:19.221+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:19.221] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:20.221+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:20.221] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:21.227+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:21.227] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:22.222+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:22.222] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:23.223+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:23.223] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:23.657+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:40:23.657] clusterConfig unchanged [2021-10-22T00:40:24.224+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:24.224] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:25.225+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:25.225] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:26.225+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:26.225] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:27.226+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:27.226] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:28.226+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:28.226] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:29.226+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:29.226] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:30.227+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:30.227] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:31.227+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:31.227] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:32.228+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:32.228] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:33.148+0000] [.info] [src/director/director.go:computePlan:280] [00:40:33.148] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:40:33.148+0000] [.info] [src/director/director.go:tracef:794] [00:40:33.148] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:40:33.148+0000] [.info] [src/director/director.go:tracef:794] [00:40:33.148] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:40:33.149+0000] [.info] [src/director/director.go:tracef:794] [00:40:33.149] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:40:33.149+0000] [.info] [src/director/director.go:tracef:794] [00:40:33.149] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:40:33.149+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:40:33.149] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:40:33.231+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:33.231] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:33.301+0000] [.info] [src/director/director.go:tracef:794] [00:40:33.301] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:40:33.301+0000] [.info] [src/director/director.go:tracef:794] [00:40:33.301] because [2021-10-22T00:40:33.301+0000] [.info] [src/director/director.go:tracef:794] [00:40:33.301] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:40:33.301+0000] [.info] [src/director/director.go:tracef:794] [00:40:33.301] because ['currentState.CanBounceStop' = true] [2021-10-22T00:40:33.456+0000] [.info] [src/director/director.go:tracef:794] [00:40:33.456] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:40:33.456+0000] [.info] [src/director/director.go:tracef:794] [00:40:33.456] because [2021-10-22T00:40:33.456+0000] [.info] [src/director/director.go:tracef:794] [00:40:33.456] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:40:33.456+0000] [.info] [src/director/director.go:tracef:794] [00:40:33.456] because ['currentState.Up' = false] [2021-10-22T00:40:33.456+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:40:33.456] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:40:33.456+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:40:33.456] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:40:33.544+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:40:33.544] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:40:33.552+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:40:33.552] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:40:33.553+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:40:33.553] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:40:33.554+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:40:33.554] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:40:33.554+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:40:33.554] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:40:33.554+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:40:33.554] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:40:33.556+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:40:33.556] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:40:33.784+0000] [.info] [src/director/director.go:tracef:794] [00:40:33.784] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:40:33.784+0000] [.info] [src/director/director.go:tracef:794] [00:40:33.784] because ['currentState.Up' = false] [2021-10-22T00:40:33.860+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:40:33.860] clusterConfig unchanged [2021-10-22T00:40:34.006+0000] [.info] [src/director/director.go:tracef:794] [00:40:34.006] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:40:34.006+0000] [.info] [src/director/director.go:tracef:794] [00:40:34.006] because ['currentState.Up' = false] [2021-10-22T00:40:34.006+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:40:34.006] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:40:34.195+0000] [.info] [src/director/director.go:tracef:794] [00:40:34.195] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:40:34.195+0000] [.info] [src/director/director.go:tracef:794] [00:40:34.195] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:40:34.247+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:34.247] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:34.416+0000] [.info] [src/director/director.go:tracef:794] [00:40:34.416] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:40:34.416+0000] [.info] [src/director/director.go:tracef:794] [00:40:34.416] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:40:34.416+0000] [.info] [src/director/director.go:tracef:794] [00:40:34.416] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:40:34.416+0000] [.info] [src/director/director.go:tracef:794] [00:40:34.416] because ['currentState.Up' = false] [2021-10-22T00:40:34.417+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:40:34.417] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:40:34.417+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:40:34.417] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:40:34.452+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:40:34.452] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:40:34.452+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:40:34.452] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 43944 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:40:34.996+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:40:34.996] clusterConfig unchanged [2021-10-22T00:40:35.164+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:40:35.164] ...Finished start [pid=64184] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":43944,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:40:35.238+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:35.238] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:35.312+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:40:35.312] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 147.655425ms. [2021-10-22T00:40:35.312+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:40:35.312] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:43944 (local=false) [2021-10-22T00:40:35.312+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:40:35.312] Dropping existing documents from system.replset [2021-10-22T00:40:35.467+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:40:35.467] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:40:35.470+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:40:35.470] Inserting oplogTruncateAfterPoint document [2021-10-22T00:40:35.470+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:40:35.470] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:40:35.471+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:40:35.471] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:40:35.471+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:40:35.471] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:40:35.485+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:40:35.485] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:40:35.485+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:40:35.485] Setting the minvalid document to [{_id ObjectID("61720883523370002becc2f0")} {t -1} {ts {0 1}}] [2021-10-22T00:40:35.497+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:40:35.497] Inserted [{_id ObjectID("61720883523370002becc2f0")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:40:35.497+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:40:35.497] Inserting new replica set configuration document. [2021-10-22T00:40:35.497+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:40:35.497] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:40:35.510+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:40:35.510] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:40:35.543+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:40:35.543] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:40:35.543+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:40:35.543] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:40:35.543+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:40:35.543] Initiating stopEphemeralMongo [2021-10-22T00:40:35.543+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:40:35.543] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:43944 (local=false) [2021-10-22T00:40:35.543+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:40:35.543] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:43944 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:40:36.029+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:40:36.029] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.051955ms. [2021-10-22T00:40:36.029+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:40:36.029] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:43944 (local=false) [2021-10-22T00:40:36.030+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:40:36.030] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.578µs. [2021-10-22T00:40:36.030+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:40:36.030] Successfully finished running stopEphemeralMongo [2021-10-22T00:40:36.030+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:40:36.030] Successfully stopped existing ephemeral mongod. [2021-10-22T00:40:36.030+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:40:36.030] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:40:36.030+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:40:36.030] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:40:36.068+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:40:36.068] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:40:36.069+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:40:36.069] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52023 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:40:36.129+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:40:36.129] clusterConfig unchanged [2021-10-22T00:40:36.241+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:36.241] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:37.235+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:40:37.235] clusterConfig unchanged [2021-10-22T00:40:37.237+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:37.237] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:37.250+0000] [.error] [src/action/start.go:func1:93] [103] [00:40:37.250] Error starting mongod : [00:40:37.250] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52023,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52023 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 65759 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:40:36.129+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:40:36.129+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:40:36.129+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52023,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:40:36.131+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:40:36.133+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:40:36.134+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:40:36.145+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:40:36.145+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:40:36.145+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:40:36.192+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:40:36.192+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:40:36.722+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863236:722224][65759:0x7f70b2d70bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:40:36.838+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863236:838474][65759:0x7f70b2d70bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:40:36.938+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863236:938232][65759:0x7f70b2d70bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T00:40:37.051+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863237:51408][65759:0x7f70b2d70bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:40:37.129+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863237:129824][65759:0x7f70b2d70bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:40:37.198+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863237:198273][65759:0x7f70b2d70bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:40:37.198+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863237:198349][65759:0x7f70b2d70bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:40:37.199+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:40:37.250+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:40:37.250] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:52023 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:40:37.250] Error starting mongod : [00:40:37.250] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52023,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52023 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 65759 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:40:36.129+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:40:36.129+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:40:36.129+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52023,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:40:36.131+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:40:36.133+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:40:36.134+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:40:36.145+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:40:36.145+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:40:36.145+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:40:36.192+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:40:36.192+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:40:36.722+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863236:722224][65759:0x7f70b2d70bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:40:36.838+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863236:838474][65759:0x7f70b2d70bc0], txn-recover: [W...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:40:37.251+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:40:37.251] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:40:37.250] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:52023 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:40:37.250] Error starting mongod : [00:40:37.250] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52023,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52023 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 65759 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:40:36.129+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:40:36.129+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:40:36.129+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52023,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:40:36.131+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:40:36.133+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:40:36.134+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:40:36.145+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:40:36.145+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:40:36.145+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:40:36.192+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:40:36.192+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:40:36.722+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863236:722224][65759:0x7f70b2d70bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:40:36.838+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ct... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:40:37.251+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:40:37.251] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:40:37.251+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:40:37.251] Initiating stopEphemeralMongo [2021-10-22T00:40:37.251+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:40:37.251] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:40:37.251+0000] [.error] [src/director/director.go:executePlan:962] [00:40:37.251] Failed to apply action. Result = : [00:40:37.251] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:40:37.250] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:52023 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:40:37.250] Error starting mongod : [00:40:37.250] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52023,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52023 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 65759 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:40:36.129+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:40:36.129+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:40:36.129+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52023,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:40:36.131+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:40:36.133+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:40:36.134+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:40:36.145+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:40:36.145+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:40:36.145+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:40:36.192+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:40:36.192+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:40:36.722+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863236:722224][65759:0x7f70b2d70bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:40:37.251+0000] [.error] [src/director/director.go:planAndExecute:567] [00:40:37.251] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:40:37.251] Failed to apply action. Result = : [00:40:37.251] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:40:37.250] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:52023 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:40:37.250] Error starting mongod : [00:40:37.250] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52023,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52023 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 65759 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:40:36.129+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:40:36.129+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:40:36.129+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52023,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:40:36.131+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:40:36.133+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:40:36.134+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:40:36.145+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:40:36.145+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:40:36.145+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:40:36.192+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:40:36.192+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:40:36.722+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:40:37.251+0000] [.error] [src/director/director.go:mainLoop:399] [00:40:37.251] Failed to planAndExecute : [00:40:37.251] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:40:37.251] Failed to apply action. Result = : [00:40:37.251] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:40:37.250] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:52023 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:40:37.250] Error starting mongod : [00:40:37.250] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52023,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52023 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 65759 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:40:36.129+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:40:36.129+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:40:36.129+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52023,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:40:36.131+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:40:36.133+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:40:36.134+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:40:36.145+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:40:36.145+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:40:36.145+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:40:36.192+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:40:36.192+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:40:36.722+00:00"},"s":"I", "c":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:40:38.239+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:38.239] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:38.496+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:40:38.496] clusterConfig unchanged [2021-10-22T00:40:39.252+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:39.252] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:40.237+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:40.237] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:41.240+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:41.240] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:41.260+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:40:41.260] memory:rss=51523584 alloc=8701104 totalalloc=1762902504 mallocs=23171870 frees=23112185 heapinuse=14213120 heapobjects=59685 gcpausetotalns=1386377636 gcpauselastns=141758 gcnum=283 utimens=30248504000 stimens=8523553000 minflt=83261 majflt=405 [2021-10-22T00:40:42.239+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:42.239] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:43.241+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:43.241] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:44.242+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:44.242] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:45.243+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:45.243] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:45.810+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:40:45.810] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.254593423s [2021-10-22T00:40:46.240+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:46.240] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:47.241+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:47.241] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:48.242+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:48.242] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:48.588+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:40:48.588] clusterConfig unchanged [2021-10-22T00:40:49.243+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:49.243] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:50.243+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:50.243] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:51.244+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:51.244] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:52.244+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:52.244] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:53.244+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:53.244] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:54.245+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:54.245] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:55.246+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:55.246] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:56.247+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:56.247] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:57.247+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:57.247] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:58.253+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:58.253] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:40:58.729+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:40:58.729] clusterConfig unchanged [2021-10-22T00:40:59.249+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:40:59.249] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:00.251+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:00.250] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:01.249+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:01.249] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:02.251+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:02.251] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:03.251+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:03.251] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:04.252+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:04.252] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:05.252+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:05.252] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:06.252+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:06.252] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:07.253+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:07.253] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:08.254+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:08.254] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:08.851+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:41:08.851] clusterConfig unchanged [2021-10-22T00:41:09.259+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:09.259] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:10.255+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:10.255] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:11.256+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:11.256] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:12.256+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:12.256] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:13.258+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:13.258] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:13.426+0000] [.info] [src/director/director.go:computePlan:280] [00:41:13.426] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:41:13.426+0000] [.info] [src/director/director.go:tracef:794] [00:41:13.426] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:41:13.426+0000] [.info] [src/director/director.go:tracef:794] [00:41:13.426] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:41:13.426+0000] [.info] [src/director/director.go:tracef:794] [00:41:13.426] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:41:13.426+0000] [.info] [src/director/director.go:tracef:794] [00:41:13.426] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:41:13.426+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:41:13.426] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:41:13.582+0000] [.info] [src/director/director.go:tracef:794] [00:41:13.582] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:41:13.582+0000] [.info] [src/director/director.go:tracef:794] [00:41:13.582] because [2021-10-22T00:41:13.583+0000] [.info] [src/director/director.go:tracef:794] [00:41:13.583] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:41:13.583+0000] [.info] [src/director/director.go:tracef:794] [00:41:13.583] because ['currentState.CanBounceStop' = true] [2021-10-22T00:41:13.736+0000] [.info] [src/director/director.go:tracef:794] [00:41:13.736] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:41:13.736+0000] [.info] [src/director/director.go:tracef:794] [00:41:13.736] because [2021-10-22T00:41:13.736+0000] [.info] [src/director/director.go:tracef:794] [00:41:13.736] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:41:13.736+0000] [.info] [src/director/director.go:tracef:794] [00:41:13.736] because ['currentState.Up' = false] [2021-10-22T00:41:13.736+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:41:13.736] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:41:13.736+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:41:13.736] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:41:13.824+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:41:13.824] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:41:13.824+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:41:13.824] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:41:13.833+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:41:13.833] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:41:13.834+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:41:13.834] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:41:13.835+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:41:13.835] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:41:13.835+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:41:13.835] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:41:13.835+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:41:13.835] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:41:13.837+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:41:13.837] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:41:14.075+0000] [.info] [src/director/director.go:tracef:794] [00:41:14.075] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:41:14.075+0000] [.info] [src/director/director.go:tracef:794] [00:41:14.075] because ['currentState.Up' = false] [2021-10-22T00:41:14.264+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:14.264] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:14.284+0000] [.info] [src/director/director.go:tracef:794] [00:41:14.284] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:41:14.284+0000] [.info] [src/director/director.go:tracef:794] [00:41:14.284] because ['currentState.Up' = false] [2021-10-22T00:41:14.284+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:41:14.284] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:41:14.496+0000] [.info] [src/director/director.go:tracef:794] [00:41:14.496] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:41:14.496+0000] [.info] [src/director/director.go:tracef:794] [00:41:14.496] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:41:14.706+0000] [.info] [src/director/director.go:tracef:794] [00:41:14.706] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:41:14.706+0000] [.info] [src/director/director.go:tracef:794] [00:41:14.706] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:41:14.707+0000] [.info] [src/director/director.go:tracef:794] [00:41:14.706] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:41:14.707+0000] [.info] [src/director/director.go:tracef:794] [00:41:14.707] because ['currentState.Up' = false] [2021-10-22T00:41:14.707+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:41:14.707] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:41:14.707+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:41:14.707] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:41:14.742+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:41:14.742] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:41:14.742+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:41:14.742] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46016 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:41:15.261+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:15.261] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:15.460+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:41:15.460] ...Finished start [pid=10915] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46016,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:41:15.572+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:41:15.572] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 111.768498ms. [2021-10-22T00:41:15.572+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:41:15.572] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46016 (local=false) [2021-10-22T00:41:15.572+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:41:15.572] Dropping existing documents from system.replset [2021-10-22T00:41:15.730+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:41:15.730] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:41:15.733+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:41:15.733] Inserting oplogTruncateAfterPoint document [2021-10-22T00:41:15.733+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:41:15.733] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:41:15.734+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:41:15.734] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:41:15.734+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:41:15.734] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:41:15.746+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:41:15.746] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:41:15.746+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:41:15.746] Setting the minvalid document to [{_id ObjectID("617208ab523370002becc2fe")} {t -1} {ts {0 1}}] [2021-10-22T00:41:15.757+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:41:15.757] Inserted [{_id ObjectID("617208ab523370002becc2fe")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:41:15.758+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:41:15.757] Inserting new replica set configuration document. [2021-10-22T00:41:15.758+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:41:15.758] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:41:15.769+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:41:15.769] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:41:15.783+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:41:15.783] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:41:15.783+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:41:15.783] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:41:15.783+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:41:15.783] Initiating stopEphemeralMongo [2021-10-22T00:41:15.783+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:41:15.783] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46016 (local=false) [2021-10-22T00:41:15.783+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:41:15.783] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46016 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:41:16.222+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:41:16.222] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.487183ms. [2021-10-22T00:41:16.222+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:41:16.222] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46016 (local=false) [2021-10-22T00:41:16.222+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:41:16.222] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.684µs. [2021-10-22T00:41:16.222+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:41:16.222] Successfully finished running stopEphemeralMongo [2021-10-22T00:41:16.222+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:41:16.222] Successfully stopped existing ephemeral mongod. [2021-10-22T00:41:16.222+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:41:16.222] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:41:16.222+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:41:16.222] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:41:16.264+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:16.264] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:16.265+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:41:16.265] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:41:16.267+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:41:16.267] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40325 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:41:17.264+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:17.264] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:17.439+0000] [.error] [src/action/start.go:func1:93] [103] [00:41:17.439] Error starting mongod : [00:41:17.438] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40325,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40325 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:41:16.299Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 12496 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:41:16.319+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:41:16.319+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:41:16.319+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40325,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:41:16.321+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:41:16.323+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:41:16.323+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:41:16.348+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:41:16.348+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:41:16.349+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:41:16.401+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:41:16.401+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:41:16.927+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863276:927853][12496:0x7f1cb3001bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:41:17.032+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863277:32155][12496:0x7f1cb3001bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:41:17.135+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863277:135500][12496:0x7f1cb3001bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T00:41:17.256+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863277:256464][12496:0x7f1cb3001bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:41:17.327+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863277:327156][12496:0x7f1cb3001bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:41:17.386+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863277:386166][12496:0x7f1cb3001bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:41:17.386+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863277:386238][12496:0x7f1cb3001bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:41:17.439+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:41:17.439] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:40325 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:41:17.439] Error starting mongod : [00:41:17.438] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40325,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40325 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:41:16.299Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 12496 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:41:16.319+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:41:16.319+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:41:16.319+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40325,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:41:16.321+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:41:16.323+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:41:16.323+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:41:16.348+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:41:16.348+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:41:16.349+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:41:16.401+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:41:16.401+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:41:16.927+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863276:927853][12496:0x7f1cb3001bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:41:17.032+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:41:17.439+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:41:17.439] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:41:17.439] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:40325 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:41:17.439] Error starting mongod : [00:41:17.438] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40325,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40325 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:41:16.299Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 12496 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:41:16.319+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:41:16.319+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:41:16.319+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40325,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:41:16.321+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:41:16.323+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:41:16.323+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:41:16.348+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:41:16.348+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:41:16.349+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:41:16.401+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:41:16.401+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:41:16.927+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863276:927853][12496:0x7f1cb3001bc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:41:17.439+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:41:17.439] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:41:17.439+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:41:17.439] Initiating stopEphemeralMongo [2021-10-22T00:41:17.439+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:41:17.439] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:41:17.440+0000] [.error] [src/director/director.go:executePlan:962] [00:41:17.440] Failed to apply action. Result = : [00:41:17.439] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:41:17.439] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:40325 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:41:17.439] Error starting mongod : [00:41:17.438] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40325,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40325 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:41:16.299Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 12496 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:41:16.319+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:41:16.319+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:41:16.319+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40325,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:41:16.321+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:41:16.323+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:41:16.323+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:41:16.348+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:41:16.348+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:41:16.349+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:41:16.401+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:41:16.401+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:41:16.927+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:41:17.440+0000] [.error] [src/director/director.go:planAndExecute:567] [00:41:17.440] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:41:17.440] Failed to apply action. Result = : [00:41:17.439] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:41:17.439] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:40325 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:41:17.439] Error starting mongod : [00:41:17.438] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40325,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40325 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:41:16.299Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 12496 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:41:16.319+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:41:16.319+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:41:16.319+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40325,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:41:16.321+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:41:16.323+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:41:16.323+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:41:16.348+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:41:16.348+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:41:16.349+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:41:16.401+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:41:16.401+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:41:17.440+0000] [.error] [src/director/director.go:mainLoop:399] [00:41:17.440] Failed to planAndExecute : [00:41:17.440] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:41:17.440] Failed to apply action. Result = : [00:41:17.439] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:41:17.439] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:40325 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:41:17.439] Error starting mongod : [00:41:17.438] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40325,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40325 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:41:16.299Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 12496 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:41:16.319+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:41:16.319+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:41:16.319+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40325,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:41:16.321+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:41:16.323+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:41:16.323+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:41:16.348+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:41:16.348+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:41:16.349+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:41:16.401+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:41:16.401+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:41:18.263+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:18.263] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:18.555+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [00:41:18.555] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T00:41:18.566+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:18.566] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T00:41:18.566+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [00:41:18.566] Could not get version : [00:41:18.566] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T00:41:18.566+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [00:41:18.566] Error getting version for log lines. Skipping : [00:41:18.566] Could not get version : [00:41:18.566] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T00:41:19.121+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:41:19.121] clusterConfig unchanged [2021-10-22T00:41:19.261+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:19.261] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:20.262+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:20.262] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:21.263+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:21.263] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:22.263+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:22.263] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:23.263+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:23.263] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:24.267+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:24.267] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:25.264+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:25.264] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:26.055+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:41:26.055] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.218141046s [2021-10-22T00:41:26.262+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:26.262] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:27.263+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:27.263] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:28.265+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:28.265] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:29.242+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:41:29.242] clusterConfig unchanged [2021-10-22T00:41:29.265+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:29.265] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:30.265+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:30.265] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:31.266+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:31.266] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:32.267+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:32.267] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:33.268+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:33.268] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:34.268+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:34.268] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:35.269+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:35.269] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:36.270+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:36.270] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:37.270+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:37.270] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:38.270+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:38.270] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:39.271+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:39.271] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:39.375+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:41:39.375] clusterConfig unchanged [2021-10-22T00:41:40.277+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:40.277] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:41.260+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:41:41.260] memory:rss=49278976 alloc=6743984 totalalloc=1849599544 mallocs=24235034 frees=24194178 heapinuse=13279232 heapobjects=40856 gcpausetotalns=1389727793 gcpauselastns=53349 gcnum=297 utimens=31724859000 stimens=8996330000 minflt=87902 majflt=405 [2021-10-22T00:41:41.273+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:41.273] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:42.273+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:42.273] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:43.273+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:43.273] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:44.130+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [00:41:44.130] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40325 (local=false) because it is not up [2021-10-22T00:41:44.274+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:44.274] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:45.274+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:45.274] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:46.279+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:46.279] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:47.278+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:47.278] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:48.276+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:48.276] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:49.277+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:49.277] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:49.467+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:41:49.467] clusterConfig unchanged [2021-10-22T00:41:50.277+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:50.277] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:51.278+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:51.278] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:52.279+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:52.279] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:53.279+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:53.279] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:54.280+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:54.280] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:55.280+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:55.280] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:56.282+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:56.282] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:57.281+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:57.281] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:58.287+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:58.287] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:58.605+0000] [.info] [src/director/director.go:computePlan:280] [00:41:58.605] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:41:58.605+0000] [.info] [src/director/director.go:tracef:794] [00:41:58.605] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:41:58.605+0000] [.info] [src/director/director.go:tracef:794] [00:41:58.605] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:41:58.605+0000] [.info] [src/director/director.go:tracef:794] [00:41:58.605] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:41:58.605+0000] [.info] [src/director/director.go:tracef:794] [00:41:58.605] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:41:58.605+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:41:58.605] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:41:58.755+0000] [.info] [src/director/director.go:tracef:794] [00:41:58.755] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:41:58.755+0000] [.info] [src/director/director.go:tracef:794] [00:41:58.755] because [2021-10-22T00:41:58.755+0000] [.info] [src/director/director.go:tracef:794] [00:41:58.755] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:41:58.755+0000] [.info] [src/director/director.go:tracef:794] [00:41:58.755] because ['currentState.CanBounceStop' = true] [2021-10-22T00:41:58.904+0000] [.info] [src/director/director.go:tracef:794] [00:41:58.904] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:41:58.904+0000] [.info] [src/director/director.go:tracef:794] [00:41:58.904] because [2021-10-22T00:41:58.904+0000] [.info] [src/director/director.go:tracef:794] [00:41:58.904] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:41:58.904+0000] [.info] [src/director/director.go:tracef:794] [00:41:58.904] because ['currentState.Up' = false] [2021-10-22T00:41:58.904+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:41:58.904] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:41:58.904+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:41:58.904] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:41:58.989+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:41:58.989] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:41:58.997+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:41:58.997] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:41:58.998+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:41:58.998] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:41:59.000+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:41:59.000] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:41:59.001+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:41:59.001] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:41:59.002+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:41:59.002] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:41:59.003+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:41:59.003] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:41:59.005+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:41:59.005] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:41:59.210+0000] [.info] [src/director/director.go:tracef:794] [00:41:59.210] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:41:59.210+0000] [.info] [src/director/director.go:tracef:794] [00:41:59.210] because ['currentState.Up' = false] [2021-10-22T00:41:59.301+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:41:59.301] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:41:59.427+0000] [.info] [src/director/director.go:tracef:794] [00:41:59.427] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:41:59.428+0000] [.info] [src/director/director.go:tracef:794] [00:41:59.428] because ['currentState.Up' = false] [2021-10-22T00:41:59.428+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:41:59.428] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:41:59.626+0000] [.info] [src/director/director.go:tracef:794] [00:41:59.626] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:41:59.626+0000] [.info] [src/director/director.go:tracef:794] [00:41:59.626] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:41:59.680+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:41:59.680] clusterConfig unchanged [2021-10-22T00:41:59.836+0000] [.info] [src/director/director.go:tracef:794] [00:41:59.836] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:41:59.837+0000] [.info] [src/director/director.go:tracef:794] [00:41:59.837] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:41:59.837+0000] [.info] [src/director/director.go:tracef:794] [00:41:59.837] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:41:59.837+0000] [.info] [src/director/director.go:tracef:794] [00:41:59.837] because ['currentState.Up' = false] [2021-10-22T00:41:59.837+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:41:59.837] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:41:59.837+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:41:59.837] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:41:59.868+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:41:59.868] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:41:59.869+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:41:59.869] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45269 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:42:00.289+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:00.289] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:00.591+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:42:00.591] ...Finished start [pid=31601] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45269,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:42:00.717+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:42:00.717] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 125.713243ms. [2021-10-22T00:42:00.717+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:42:00.717] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45269 (local=false) [2021-10-22T00:42:00.717+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:42:00.717] Dropping existing documents from system.replset [2021-10-22T00:42:00.809+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:42:00.809] clusterConfig unchanged [2021-10-22T00:42:00.832+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:42:00.832] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:42:00.838+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:42:00.838] Inserting oplogTruncateAfterPoint document [2021-10-22T00:42:00.838+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:42:00.838] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:42:00.839+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:42:00.839] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:42:00.840+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:42:00.839] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:42:00.854+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:42:00.854] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:42:00.854+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:42:00.854] Setting the minvalid document to [{_id ObjectID("617208d8523370002becc30c")} {t -1} {ts {0 1}}] [2021-10-22T00:42:00.867+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:42:00.867] Inserted [{_id ObjectID("617208d8523370002becc30c")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:42:00.867+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:42:00.867] Inserting new replica set configuration document. [2021-10-22T00:42:00.867+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:42:00.867] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:42:00.880+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:42:00.880] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:42:00.885+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:42:00.885] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:42:00.885+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:42:00.885] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:42:00.885+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:42:00.885] Initiating stopEphemeralMongo [2021-10-22T00:42:00.885+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:42:00.885] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45269 (local=false) [2021-10-22T00:42:00.885+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:42:00.885] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45269 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:42:01.231+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:42:01.231] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.351262ms. [2021-10-22T00:42:01.231+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:42:01.231] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45269 (local=false) [2021-10-22T00:42:01.231+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:42:01.231] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 38.5µs. [2021-10-22T00:42:01.231+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:42:01.231] Successfully finished running stopEphemeralMongo [2021-10-22T00:42:01.231+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:42:01.231] Successfully stopped existing ephemeral mongod. [2021-10-22T00:42:01.231+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:42:01.231] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:42:01.231+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:42:01.231] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:42:01.267+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:42:01.267] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:42:01.267+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:42:01.267] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58911 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:42:01.297+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:01.297] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:01.947+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:42:01.947] clusterConfig unchanged [2021-10-22T00:42:02.301+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:02.301] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:02.503+0000] [.error] [src/action/start.go:func1:93] [103] [00:42:02.503] Error starting mongod : [00:42:02.503] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58911 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:42:01.319Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 33061 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:42:01.339+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:42:01.339+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:42:01.339+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:42:01.342+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:01.343+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:42:01.344+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:42:01.360+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:01.360+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:42:01.360+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:01.405+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:01.405+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:42:01.941+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863321:941498][33061:0x7f88403eebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:42:02.054+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863322:54406][33061:0x7f88403eebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:42:02.162+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863322:162344][33061:0x7f88403eebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:42:02.284+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863322:284074][33061:0x7f88403eebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:42:02.380+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863322:380480][33061:0x7f88403eebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:42:02.452+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863322:452602][33061:0x7f88403eebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:42:02.452+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863322:452689][33061:0x7f88403eebc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:42:02.504+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:42:02.503] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:58911 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:42:02.503] Error starting mongod : [00:42:02.503] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58911 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:42:01.319Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 33061 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:42:01.339+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:42:01.339+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:42:01.339+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:42:01.342+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:01.343+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:42:01.344+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:42:01.360+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:01.360+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:42:01.360+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:01.405+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:01.405+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:42:01.941+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863321:941498][33061:0x7f88403eebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:42:02.054+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:42:02.504+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:42:02.504] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:42:02.503] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:58911 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:42:02.503] Error starting mongod : [00:42:02.503] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58911 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:42:01.319Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 33061 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:42:01.339+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:42:01.339+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:42:01.339+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:42:01.342+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:01.343+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:42:01.344+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:42:01.360+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:01.360+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:42:01.360+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:01.405+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:01.405+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:42:01.941+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863321:941498][33061:0x7f88403eebc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:42:02.504+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:42:02.504] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:42:02.504+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:42:02.504] Initiating stopEphemeralMongo [2021-10-22T00:42:02.504+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:42:02.504] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:42:02.505+0000] [.error] [src/director/director.go:executePlan:962] [00:42:02.505] Failed to apply action. Result = : [00:42:02.504] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:42:02.503] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:58911 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:42:02.503] Error starting mongod : [00:42:02.503] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58911 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:42:01.319Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 33061 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:42:01.339+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:42:01.339+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:42:01.339+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:42:01.342+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:01.343+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:42:01.344+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:42:01.360+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:01.360+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:42:01.360+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:01.405+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:01.405+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:42:01.941+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:42:02.505+0000] [.error] [src/director/director.go:planAndExecute:567] [00:42:02.505] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:42:02.505] Failed to apply action. Result = : [00:42:02.504] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:42:02.503] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:58911 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:42:02.503] Error starting mongod : [00:42:02.503] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58911 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:42:01.319Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 33061 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:42:01.339+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:42:01.339+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:42:01.339+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:42:01.342+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:01.343+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:42:01.344+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:42:01.360+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:01.360+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:42:01.360+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:01.405+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:01.405+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:42:02.505+0000] [.error] [src/director/director.go:mainLoop:399] [00:42:02.505] Failed to planAndExecute : [00:42:02.505] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:42:02.505] Failed to apply action. Result = : [00:42:02.504] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:42:02.503] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:58911 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:42:02.503] Error starting mongod : [00:42:02.503] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58911 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:42:01.319Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 33061 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:42:01.339+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:42:01.339+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:42:01.339+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:42:01.342+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:01.343+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:42:01.344+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:42:01.360+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:01.360+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:42:01.360+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:01.405+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:01.405+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:42:03.172+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:42:03.172] clusterConfig unchanged [2021-10-22T00:42:03.313+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:03.313] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:04.302+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:04.302] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:05.303+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:05.303] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:06.302+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:06.302] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:07.305+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:07.305] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:08.304+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:08.304] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:09.313+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:09.313] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:10.308+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:10.308] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:11.292+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:42:11.292] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.286549353s [2021-10-22T00:42:11.305+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:11.305] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:12.305+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:12.305] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:13.295+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:42:13.295] clusterConfig unchanged [2021-10-22T00:42:13.306+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:13.306] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:14.306+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:14.306] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:15.307+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:15.307] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:16.307+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:16.307] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:17.308+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:17.308] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:18.309+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:18.309] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:19.309+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:19.309] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:20.310+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:20.310] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:21.310+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:21.310] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:22.315+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:22.315] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:23.312+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:23.312] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:23.436+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:42:23.436] clusterConfig unchanged [2021-10-22T00:42:24.312+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:24.312] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:25.313+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:25.313] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:26.314+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:26.314] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:27.314+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:27.314] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:27.556+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [00:42:27.556] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T00:42:27.559+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [00:42:27.559] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T00:42:28.319+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:28.319] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:29.315+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:29.315] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:30.316+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:30.316] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:31.317+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:31.317] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:32.317+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:32.317] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:33.318+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:33.318] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:33.548+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:42:33.548] clusterConfig unchanged [2021-10-22T00:42:34.320+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [00:42:34.320] lsblk cache invalidated, running lsblk command [2021-10-22T00:42:34.321+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:34.321] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:35.323+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:35.323] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:36.319+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:36.319] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:37.319+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:37.319] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:38.320+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:38.320] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:39.321+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:39.321] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:40.321+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:40.321] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:41.262+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:42:41.262] memory:rss=49926144 alloc=9530664 totalalloc=1934646608 mallocs=25300595 frees=25230594 heapinuse=14090240 heapobjects=70001 gcpausetotalns=1391304604 gcpauselastns=64444 gcnum=310 utimens=33156392000 stimens=9460779000 minflt=91795 majflt=405 [2021-10-22T00:42:41.323+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:41.323] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:42.323+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:42.323] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:43.327+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:43.327] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:43.671+0000] [.info] [src/director/director.go:computePlan:280] [00:42:43.671] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:42:43.671+0000] [.info] [src/director/director.go:tracef:794] [00:42:43.671] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:42:43.671+0000] [.info] [src/director/director.go:tracef:794] [00:42:43.671] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:42:43.672+0000] [.info] [src/director/director.go:tracef:794] [00:42:43.671] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:42:43.672+0000] [.info] [src/director/director.go:tracef:794] [00:42:43.672] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:42:43.672+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:42:43.672] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:42:43.748+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:42:43.748] clusterConfig unchanged [2021-10-22T00:42:43.826+0000] [.info] [src/director/director.go:tracef:794] [00:42:43.826] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:42:43.826+0000] [.info] [src/director/director.go:tracef:794] [00:42:43.826] because [2021-10-22T00:42:43.826+0000] [.info] [src/director/director.go:tracef:794] [00:42:43.826] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:42:43.826+0000] [.info] [src/director/director.go:tracef:794] [00:42:43.826] because ['currentState.CanBounceStop' = true] [2021-10-22T00:42:43.979+0000] [.info] [src/director/director.go:tracef:794] [00:42:43.979] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:42:43.979+0000] [.info] [src/director/director.go:tracef:794] [00:42:43.979] because [2021-10-22T00:42:43.979+0000] [.info] [src/director/director.go:tracef:794] [00:42:43.979] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:42:43.979+0000] [.info] [src/director/director.go:tracef:794] [00:42:43.979] because ['currentState.Up' = false] [2021-10-22T00:42:43.979+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:42:43.979] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:42:43.979+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:42:43.979] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:42:44.065+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:42:44.065] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:42:44.075+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:42:44.075] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:42:44.076+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:42:44.076] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:42:44.076+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:42:44.076] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:42:44.078+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:42:44.078] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:42:44.078+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:42:44.078] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:42:44.078+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:42:44.078] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:42:44.080+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:42:44.080] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:42:44.283+0000] [.info] [src/director/director.go:tracef:794] [00:42:44.283] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:42:44.283+0000] [.info] [src/director/director.go:tracef:794] [00:42:44.283] because ['currentState.Up' = false] [2021-10-22T00:42:44.326+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:44.326] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:44.496+0000] [.info] [src/director/director.go:tracef:794] [00:42:44.496] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:42:44.496+0000] [.info] [src/director/director.go:tracef:794] [00:42:44.496] because ['currentState.Up' = false] [2021-10-22T00:42:44.496+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:42:44.496] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:42:44.699+0000] [.info] [src/director/director.go:tracef:794] [00:42:44.699] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:42:44.699+0000] [.info] [src/director/director.go:tracef:794] [00:42:44.699] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:42:44.988+0000] [.info] [src/director/director.go:tracef:794] [00:42:44.988] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:42:44.988+0000] [.info] [src/director/director.go:tracef:794] [00:42:44.988] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:42:44.988+0000] [.info] [src/director/director.go:tracef:794] [00:42:44.988] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:42:44.988+0000] [.info] [src/director/director.go:tracef:794] [00:42:44.988] because ['currentState.Up' = false] [2021-10-22T00:42:44.988+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:42:44.988] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:42:44.988+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:42:44.988] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:42:45.024+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:42:45.024] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:42:45.025+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:42:45.025] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46861 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:42:45.328+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:45.328] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:45.750+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:42:45.750] ...Finished start [pid=52510] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46861,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:42:45.876+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:42:45.876] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 126.014323ms. [2021-10-22T00:42:45.876+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:42:45.876] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46861 (local=false) [2021-10-22T00:42:45.876+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:42:45.876] Dropping existing documents from system.replset [2021-10-22T00:42:46.039+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:42:46.039] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:42:46.043+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:42:46.043] Inserting oplogTruncateAfterPoint document [2021-10-22T00:42:46.043+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:42:46.043] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:42:46.044+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:42:46.044] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:42:46.044+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:42:46.044] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:42:46.057+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:42:46.057] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:42:46.057+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:42:46.057] Setting the minvalid document to [{_id ObjectID("61720906523370002becc31a")} {t -1} {ts {0 1}}] [2021-10-22T00:42:46.069+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:42:46.069] Inserted [{_id ObjectID("61720906523370002becc31a")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:42:46.069+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:42:46.069] Inserting new replica set configuration document. [2021-10-22T00:42:46.070+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:42:46.069] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:42:46.082+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:42:46.082] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:42:46.085+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:42:46.085] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:42:46.085+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:42:46.085] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:42:46.085+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:42:46.085] Initiating stopEphemeralMongo [2021-10-22T00:42:46.085+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:42:46.085] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46861 (local=false) [2021-10-22T00:42:46.085+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:42:46.085] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46861 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:42:46.326+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:46.326] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:46.480+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:42:46.480] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.605534ms. [2021-10-22T00:42:46.480+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:42:46.480] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46861 (local=false) [2021-10-22T00:42:46.480+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:42:46.480] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 38.904µs. [2021-10-22T00:42:46.480+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:42:46.480] Successfully finished running stopEphemeralMongo [2021-10-22T00:42:46.480+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:42:46.480] Successfully stopped existing ephemeral mongod. [2021-10-22T00:42:46.480+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:42:46.480] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:42:46.480+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:42:46.480] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:42:46.517+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:42:46.517] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:42:46.519+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:42:46.519] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56888 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:42:47.329+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:47.329] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:47.719+0000] [.error] [src/action/start.go:func1:93] [103] [00:42:47.719] Error starting mongod : [00:42:47.719] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56888,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56888 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:42:46.556Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 54090 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:42:46.574+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:42:46.574+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:42:46.574+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56888,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:42:46.576+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:46.577+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:42:46.579+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:42:46.618+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:46.618+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:42:46.619+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:46.669+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:46.669+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:42:47.197+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863367:197406][54090:0x7f6b2b68cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:42:47.303+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863367:303828][54090:0x7f6b2b68cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:42:47.403+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863367:403243][54090:0x7f6b2b68cbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:42:47.529+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863367:529300][54090:0x7f6b2b68cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:42:47.606+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863367:606422][54090:0x7f6b2b68cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:42:47.666+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863367:666510][54090:0x7f6b2b68cbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:42:47.666+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863367:666585][54090:0x7f6b2b68cbc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:42:47.719+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:42:47.719] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:56888 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:42:47.719] Error starting mongod : [00:42:47.719] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56888,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56888 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:42:46.556Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 54090 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:42:46.574+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:42:46.574+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:42:46.574+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56888,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:42:46.576+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:46.577+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:42:46.579+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:42:46.618+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:46.618+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:42:46.619+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:46.669+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:46.669+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:42:47.197+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863367:197406][54090:0x7f6b2b68cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:42:47.303+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:42:47.719+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:42:47.719] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:42:47.719] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:56888 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:42:47.719] Error starting mongod : [00:42:47.719] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56888,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56888 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:42:46.556Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 54090 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:42:46.574+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:42:46.574+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:42:46.574+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56888,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:42:46.576+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:46.577+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:42:46.579+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:42:46.618+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:46.618+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:42:46.619+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:46.669+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:46.669+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:42:47.197+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863367:197406][54090:0x7f6b2b68cbc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:42:47.720+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:42:47.719] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:42:47.720+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:42:47.720] Initiating stopEphemeralMongo [2021-10-22T00:42:47.720+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:42:47.720] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:42:47.720+0000] [.error] [src/director/director.go:executePlan:962] [00:42:47.720] Failed to apply action. Result = : [00:42:47.719] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:42:47.719] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:56888 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:42:47.719] Error starting mongod : [00:42:47.719] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56888,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56888 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:42:46.556Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 54090 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:42:46.574+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:42:46.574+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:42:46.574+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56888,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:42:46.576+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:46.577+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:42:46.579+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:42:46.618+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:46.618+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:42:46.619+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:46.669+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:46.669+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:42:47.197+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:42:47.720+0000] [.error] [src/director/director.go:planAndExecute:567] [00:42:47.720] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:42:47.720] Failed to apply action. Result = : [00:42:47.719] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:42:47.719] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:56888 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:42:47.719] Error starting mongod : [00:42:47.719] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56888,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56888 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:42:46.556Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 54090 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:42:46.574+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:42:46.574+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:42:46.574+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56888,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:42:46.576+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:46.577+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:42:46.579+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:42:46.618+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:46.618+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:42:46.619+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:46.669+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:46.669+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:42:47.720+0000] [.error] [src/director/director.go:mainLoop:399] [00:42:47.720] Failed to planAndExecute : [00:42:47.720] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:42:47.720] Failed to apply action. Result = : [00:42:47.719] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:42:47.719] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:56888 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:42:47.719] Error starting mongod : [00:42:47.719] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56888,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56888 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:42:46.556Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 54090 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:42:46.574+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:42:46.574+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:42:46.574+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56888,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:42:46.576+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:46.577+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:42:46.579+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:42:46.618+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:46.618+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:42:46.619+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:42:46.669+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:42:46.669+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:42:48.329+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:48.329] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:49.328+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:49.328] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:50.330+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:50.330] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:51.332+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:51.332] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:52.332+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:52.332] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:53.332+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:53.332] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:53.998+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:42:53.998] clusterConfig unchanged [2021-10-22T00:42:54.333+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:54.333] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:55.335+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:55.335] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:56.263+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:42:56.263] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.183237395s [2021-10-22T00:42:56.332+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:56.332] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:57.333+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:57.333] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:58.333+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:58.333] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:42:59.334+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:42:59.334] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:00.334+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:00.334] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:01.339+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:01.339] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:02.334+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:02.334] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:03.335+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:03.335] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:04.130+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:43:04.130] clusterConfig unchanged [2021-10-22T00:43:04.336+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:04.336] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:05.337+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:05.337] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:06.337+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:06.337] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:07.337+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:07.337] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:08.338+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:08.338] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:09.339+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:09.338] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:10.358+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:10.358] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:11.339+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:11.339] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:12.341+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:12.341] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:13.346+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:13.346] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:14.261+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:43:14.261] clusterConfig unchanged [2021-10-22T00:43:14.341+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:14.341] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:15.342+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:15.342] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:16.351+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:16.351] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:17.343+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:17.343] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:18.342+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:18.342] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:19.343+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:19.343] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:20.343+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:20.343] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:21.344+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:21.344] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:22.345+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:22.345] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:23.351+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:23.351] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:24.346+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:24.346] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:24.373+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:43:24.373] clusterConfig unchanged [2021-10-22T00:43:25.352+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:25.352] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:26.347+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:26.347] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:27.348+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:27.348] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:28.349+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:28.349] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:29.350+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:29.350] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:30.351+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:30.351] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:31.352+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:31.352] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:32.353+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:32.353] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:33.353+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:33.353] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:33.886+0000] [.info] [src/director/director.go:computePlan:280] [00:43:33.886] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:43:33.887+0000] [.info] [src/director/director.go:tracef:794] [00:43:33.887] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:43:33.887+0000] [.info] [src/director/director.go:tracef:794] [00:43:33.887] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:43:33.887+0000] [.info] [src/director/director.go:tracef:794] [00:43:33.887] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:43:33.887+0000] [.info] [src/director/director.go:tracef:794] [00:43:33.887] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:43:33.887+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:43:33.887] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:43:34.047+0000] [.info] [src/director/director.go:tracef:794] [00:43:34.047] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:43:34.047+0000] [.info] [src/director/director.go:tracef:794] [00:43:34.047] because [2021-10-22T00:43:34.047+0000] [.info] [src/director/director.go:tracef:794] [00:43:34.047] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:43:34.047+0000] [.info] [src/director/director.go:tracef:794] [00:43:34.047] because ['currentState.CanBounceStop' = true] [2021-10-22T00:43:34.202+0000] [.info] [src/director/director.go:tracef:794] [00:43:34.202] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:43:34.202+0000] [.info] [src/director/director.go:tracef:794] [00:43:34.202] because [2021-10-22T00:43:34.202+0000] [.info] [src/director/director.go:tracef:794] [00:43:34.202] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:43:34.202+0000] [.info] [src/director/director.go:tracef:794] [00:43:34.202] because ['currentState.Up' = false] [2021-10-22T00:43:34.202+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:43:34.202] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:43:34.202+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:43:34.202] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:43:34.328+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:43:34.328] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:43:34.337+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:43:34.337] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:43:34.337+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:43:34.337] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:43:34.338+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:43:34.338] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:43:34.339+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:43:34.339] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:43:34.339+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:43:34.339] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:43:34.339+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:43:34.339] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:43:34.340+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:43:34.340] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:43:34.358+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:34.358] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:34.565+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:43:34.565] clusterConfig unchanged [2021-10-22T00:43:34.650+0000] [.info] [src/director/director.go:tracef:794] [00:43:34.649] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:43:34.650+0000] [.info] [src/director/director.go:tracef:794] [00:43:34.650] because ['currentState.Up' = false] [2021-10-22T00:43:34.901+0000] [.info] [src/director/director.go:tracef:794] [00:43:34.901] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:43:34.901+0000] [.info] [src/director/director.go:tracef:794] [00:43:34.901] because ['currentState.Up' = false] [2021-10-22T00:43:34.901+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:43:34.901] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:43:35.128+0000] [.info] [src/director/director.go:tracef:794] [00:43:35.128] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:43:35.128+0000] [.info] [src/director/director.go:tracef:794] [00:43:35.128] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:43:35.337+0000] [.info] [src/director/director.go:tracef:794] [00:43:35.337] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:43:35.337+0000] [.info] [src/director/director.go:tracef:794] [00:43:35.337] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:43:35.338+0000] [.info] [src/director/director.go:tracef:794] [00:43:35.338] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:43:35.338+0000] [.info] [src/director/director.go:tracef:794] [00:43:35.338] because ['currentState.Up' = false] [2021-10-22T00:43:35.338+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:43:35.338] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:43:35.338+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:43:35.338] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:43:35.358+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:35.358] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:35.374+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:43:35.374] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:43:35.374+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:43:35.374] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53089 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:43:35.697+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:43:35.697] clusterConfig unchanged [2021-10-22T00:43:36.120+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:43:36.120] ...Finished start [pid=72947] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53089,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:43:36.241+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:43:36.241] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 121.182758ms. [2021-10-22T00:43:36.241+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:43:36.241] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53089 (local=false) [2021-10-22T00:43:36.241+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:43:36.241] Dropping existing documents from system.replset [2021-10-22T00:43:36.358+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:36.358] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:36.367+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:43:36.367] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:43:36.372+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:43:36.372] Inserting oplogTruncateAfterPoint document [2021-10-22T00:43:36.372+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:43:36.372] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:43:36.373+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:43:36.373] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:43:36.373+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:43:36.373] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:43:36.384+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:43:36.384] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:43:36.384+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:43:36.384] Setting the minvalid document to [{_id ObjectID("61720938523370002becc327")} {t -1} {ts {0 1}}] [2021-10-22T00:43:36.398+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:43:36.398] Inserted [{_id ObjectID("61720938523370002becc327")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:43:36.398+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:43:36.398] Inserting new replica set configuration document. [2021-10-22T00:43:36.398+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:43:36.398] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:43:36.410+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:43:36.410] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:43:36.414+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:43:36.414] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:43:36.414+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:43:36.414] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:43:36.414+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:43:36.414] Initiating stopEphemeralMongo [2021-10-22T00:43:36.414+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:43:36.414] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53089 (local=false) [2021-10-22T00:43:36.414+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:43:36.414] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53089 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:43:36.797+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:43:36.797] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 4.080351ms. [2021-10-22T00:43:36.797+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:43:36.797] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53089 (local=false) [2021-10-22T00:43:36.797+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:43:36.797] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 39.831µs. [2021-10-22T00:43:36.797+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:43:36.797] Successfully finished running stopEphemeralMongo [2021-10-22T00:43:36.797+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:43:36.797] Successfully stopped existing ephemeral mongod. [2021-10-22T00:43:36.797+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:43:36.797] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:43:36.797+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:43:36.797] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:43:36.832+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:43:36.832] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:43:36.832+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:43:36.832] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46606 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:43:36.844+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:43:36.844] clusterConfig unchanged [2021-10-22T00:43:37.360+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:37.360] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:37.971+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:43:37.971] clusterConfig unchanged [2021-10-22T00:43:38.055+0000] [.error] [src/action/start.go:func1:93] [103] [00:43:38.055] Error starting mongod : [00:43:38.055] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46606,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46606 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 330 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: as-4wnh1p-config-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:43:36.885+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:43:36.885+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:43:36.886+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46606,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:43:36.887+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:43:36.889+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:43:36.890+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:43:36.980+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:43:36.980+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:43:36.981+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:43:36.989+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:43:36.989+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:43:37.490+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863417:490755][330:0x7f5be71d4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:43:37.594+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863417:594863][330:0x7f5be71d4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:43:37.693+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863417:693086][330:0x7f5be71d4bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:43:37.858+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863417:858454][330:0x7f5be71d4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:43:37.937+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863417:937114][330:0x7f5be71d4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:43:38.004+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863418:4275][330:0x7f5be71d4bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:43:38.004+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863418:4350][330:0x7f5be71d4bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:43:38.006+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"i...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:43:38.055+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:43:38.055] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46606 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:43:38.055] Error starting mongod : [00:43:38.055] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46606,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46606 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 330 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: as-4wnh1p-config-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:43:36.885+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:43:36.885+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:43:36.886+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46606,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:43:36.887+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:43:36.889+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:43:36.890+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:43:36.980+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:43:36.980+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:43:36.981+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:43:36.989+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:43:36.989+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:43:37.490+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863417:490755][330:0x7f5be71d4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:43:37.594+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863417:594863][330:0x7f5be71d4bc0], ...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:43:38.055+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:43:38.055] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:43:38.055] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46606 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:43:38.055] Error starting mongod : [00:43:38.055] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46606,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46606 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 330 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: as-4wnh1p-config-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:43:36.885+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:43:36.885+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:43:36.886+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46606,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:43:36.887+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:43:36.889+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:43:36.890+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:43:36.980+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:43:36.980+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:43:36.981+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:43:36.989+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:43:36.989+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:43:37.490+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863417:490755][330:0x7f5be71d4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:43:37.594+00:00"},"s":"I", "c":"STORAGE", ... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:43:38.056+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:43:38.056] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:43:38.056+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:43:38.056] Initiating stopEphemeralMongo [2021-10-22T00:43:38.056+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:43:38.056] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:43:38.056+0000] [.error] [src/director/director.go:executePlan:962] [00:43:38.056] Failed to apply action. Result = : [00:43:38.055] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:43:38.055] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46606 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:43:38.055] Error starting mongod : [00:43:38.055] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46606,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46606 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 330 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: as-4wnh1p-config-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:43:36.885+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:43:36.885+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:43:36.886+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46606,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:43:36.887+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:43:36.889+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:43:36.890+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:43:36.980+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:43:36.980+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:43:36.981+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:43:36.989+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:43:36.989+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:43:37.490+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863417:490755][330:0x7f5be71d4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:43:38.056+0000] [.error] [src/director/director.go:planAndExecute:567] [00:43:38.056] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:43:38.056] Failed to apply action. Result = : [00:43:38.055] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:43:38.055] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46606 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:43:38.055] Error starting mongod : [00:43:38.055] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46606,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46606 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 330 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: as-4wnh1p-config-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:43:36.885+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:43:36.885+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:43:36.886+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46606,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:43:36.887+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:43:36.889+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:43:36.890+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:43:36.980+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:43:36.980+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:43:36.981+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:43:36.989+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:43:36.989+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:43:37.490+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:43:38.056+0000] [.error] [src/director/director.go:mainLoop:399] [00:43:38.056] Failed to planAndExecute : [00:43:38.056] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:43:38.056] Failed to apply action. Result = : [00:43:38.055] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:43:38.055] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46606 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:43:38.055] Error starting mongod : [00:43:38.055] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46606,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46606 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 330 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: as-4wnh1p-config-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:43:36.885+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:43:36.885+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:43:36.886+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46606,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:43:36.887+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:43:36.889+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:43:36.890+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:43:36.980+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:43:36.980+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:43:36.981+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:43:36.989+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:43:36.989+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:43:37.490+00:... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:43:38.360+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:38.360] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:39.212+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:43:39.212] clusterConfig unchanged [2021-10-22T00:43:39.360+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:39.360] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:40.359+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:40.359] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:41.263+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:43:41.263] memory:rss=51613696 alloc=7111152 totalalloc=2069957240 mallocs=27051970 frees=27009670 heapinuse=14090240 heapobjects=42300 gcpausetotalns=1395305080 gcpauselastns=937187 gcnum=332 utimens=35442616000 stimens=10080477000 minflt=97145 majflt=405 [2021-10-22T00:43:41.362+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:41.362] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:42.362+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:42.362] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:43.363+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:43.363] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:44.364+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:44.364] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:45.364+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:45.364] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:46.365+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:46.365] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:46.861+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:43:46.861] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.521008589s [2021-10-22T00:43:47.367+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:47.367] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:48.368+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:48.368] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:49.333+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:43:49.333] clusterConfig unchanged [2021-10-22T00:43:49.364+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:49.364] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:50.364+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:50.364] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:51.364+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:51.364] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:52.365+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:52.365] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:53.365+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:53.365] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:54.366+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:54.366] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:55.366+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:55.366] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:56.366+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:56.366] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:57.367+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:57.367] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:58.374+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:58.374] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:59.369+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:43:59.369] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:43:59.464+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:43:59.464] clusterConfig unchanged [2021-10-22T00:44:00.371+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:00.371] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:01.371+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:01.371] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:02.372+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:02.372] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:03.372+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:03.372] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:04.373+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:04.373] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:05.374+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:05.374] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:06.375+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:06.375] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:07.375+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:07.375] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:08.376+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:08.376] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:09.376+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:09.376] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:09.586+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:44:09.586] clusterConfig unchanged [2021-10-22T00:44:10.382+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:10.382] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:11.378+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:11.378] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:12.378+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:12.378] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:13.379+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:13.379] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:14.379+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:14.379] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:15.380+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:15.380] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:16.380+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:16.380] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:17.381+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:17.381] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:18.382+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:18.382] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:19.382+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:19.382] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:19.728+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:44:19.728] clusterConfig unchanged [2021-10-22T00:44:20.383+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:20.383] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:21.383+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:21.383] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:22.384+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:22.384] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:23.384+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:23.384] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:24.221+0000] [.info] [src/director/director.go:computePlan:280] [00:44:24.221] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:44:24.221+0000] [.info] [src/director/director.go:tracef:794] [00:44:24.221] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:44:24.222+0000] [.info] [src/director/director.go:tracef:794] [00:44:24.222] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:44:24.222+0000] [.info] [src/director/director.go:tracef:794] [00:44:24.222] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:44:24.222+0000] [.info] [src/director/director.go:tracef:794] [00:44:24.222] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:44:24.222+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:44:24.222] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:44:24.385+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:24.385] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:24.386+0000] [.info] [src/director/director.go:tracef:794] [00:44:24.386] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:44:24.386+0000] [.info] [src/director/director.go:tracef:794] [00:44:24.386] because [2021-10-22T00:44:24.386+0000] [.info] [src/director/director.go:tracef:794] [00:44:24.386] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:44:24.386+0000] [.info] [src/director/director.go:tracef:794] [00:44:24.386] because ['currentState.CanBounceStop' = true] [2021-10-22T00:44:24.540+0000] [.info] [src/director/director.go:tracef:794] [00:44:24.540] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:44:24.540+0000] [.info] [src/director/director.go:tracef:794] [00:44:24.540] because [2021-10-22T00:44:24.541+0000] [.info] [src/director/director.go:tracef:794] [00:44:24.541] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:44:24.541+0000] [.info] [src/director/director.go:tracef:794] [00:44:24.541] because ['currentState.Up' = false] [2021-10-22T00:44:24.541+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:44:24.541] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:44:24.541+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:44:24.541] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:44:24.622+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:44:24.622] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:44:24.630+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:44:24.630] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:44:24.631+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:44:24.631] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:44:24.632+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:44:24.632] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:44:24.632+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:44:24.632] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:44:24.632+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:44:24.632] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:44:24.634+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:44:24.634] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:44:24.862+0000] [.info] [src/director/director.go:tracef:794] [00:44:24.862] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:44:24.862+0000] [.info] [src/director/director.go:tracef:794] [00:44:24.862] because ['currentState.Up' = false] [2021-10-22T00:44:25.056+0000] [.info] [src/director/director.go:tracef:794] [00:44:25.056] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:44:25.056+0000] [.info] [src/director/director.go:tracef:794] [00:44:25.056] because ['currentState.Up' = false] [2021-10-22T00:44:25.056+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:44:25.056] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:44:25.313+0000] [.info] [src/director/director.go:tracef:794] [00:44:25.313] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:44:25.313+0000] [.info] [src/director/director.go:tracef:794] [00:44:25.313] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:44:25.401+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:25.401] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:25.547+0000] [.info] [src/director/director.go:tracef:794] [00:44:25.547] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:44:25.547+0000] [.info] [src/director/director.go:tracef:794] [00:44:25.547] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:44:25.547+0000] [.info] [src/director/director.go:tracef:794] [00:44:25.547] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:44:25.547+0000] [.info] [src/director/director.go:tracef:794] [00:44:25.547] because ['currentState.Up' = false] [2021-10-22T00:44:25.547+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:44:25.547] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:44:25.547+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:44:25.547] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:44:25.582+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:44:25.582] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:44:25.582+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:44:25.582] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 44848 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:44:26.299+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:44:26.299] ...Finished start [pid=20008] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44848,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:44:26.388+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:26.388] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:26.422+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:44:26.422] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 122.925799ms. [2021-10-22T00:44:26.422+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:44:26.422] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44848 (local=false) [2021-10-22T00:44:26.422+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:44:26.422] Dropping existing documents from system.replset [2021-10-22T00:44:26.611+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:44:26.611] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:44:26.614+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:44:26.614] Inserting oplogTruncateAfterPoint document [2021-10-22T00:44:26.614+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:44:26.614] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:44:26.615+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:44:26.615] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:44:26.615+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:44:26.615] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:44:26.629+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:44:26.629] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:44:26.629+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:44:26.629] Setting the minvalid document to [{_id ObjectID("6172096a523370002becc335")} {t -1} {ts {0 1}}] [2021-10-22T00:44:26.642+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:44:26.642] Inserted [{_id ObjectID("6172096a523370002becc335")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:44:26.642+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:44:26.642] Inserting new replica set configuration document. [2021-10-22T00:44:26.642+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:44:26.642] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:44:26.654+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:44:26.653] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:44:26.688+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:44:26.688] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:44:26.688+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:44:26.688] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:44:26.688+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:44:26.688] Initiating stopEphemeralMongo [2021-10-22T00:44:26.688+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:44:26.688] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44848 (local=false) [2021-10-22T00:44:26.688+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:44:26.688] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44848 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:44:27.179+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:44:27.179] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.420756ms. [2021-10-22T00:44:27.179+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:44:27.179] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44848 (local=false) [2021-10-22T00:44:27.180+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:44:27.180] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 42.415µs. [2021-10-22T00:44:27.180+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:44:27.180] Successfully finished running stopEphemeralMongo [2021-10-22T00:44:27.180+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:44:27.180] Successfully stopped existing ephemeral mongod. [2021-10-22T00:44:27.180+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:44:27.180] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:44:27.180+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:44:27.180] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:44:27.218+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:44:27.218] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:44:27.219+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:44:27.219] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41211 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:44:27.389+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:27.389] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:28.390+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:28.390] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:28.447+0000] [.error] [src/action/start.go:func1:93] [103] [00:44:28.447] Error starting mongod : [00:44:28.447] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41211,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41211 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 21652 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:44:27.271+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:44:27.271+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:44:27.271+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41211,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:44:27.273+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:44:27.274+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:44:27.276+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:44:27.349+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:44:27.349+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:44:27.360+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:44:27.410+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:44:27.410+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:44:27.947+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863467:947095][21652:0x7fcd18128bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:44:28.046+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863468:46632][21652:0x7fcd18128bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:44:28.145+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863468:145519][21652:0x7fcd18128bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:44:28.256+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863468:256547][21652:0x7fcd18128bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:44:28.332+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863468:332260][21652:0x7fcd18128bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:44:28.396+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863468:396112][21652:0x7fcd18128bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:44:28.396+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863468:396179][21652:0x7fcd18128bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:44:28.397+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:44:28.447+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:44:28.447] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:41211 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:44:28.447] Error starting mongod : [00:44:28.447] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41211,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41211 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 21652 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:44:27.271+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:44:27.271+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:44:27.271+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41211,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:44:27.273+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:44:27.274+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:44:27.276+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:44:27.349+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:44:27.349+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:44:27.360+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:44:27.410+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:44:27.410+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:44:27.947+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863467:947095][21652:0x7fcd18128bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:44:28.046+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863468:46632][21652:0x7fcd18128bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:44:28.447+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:44:28.447] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:44:28.447] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:41211 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:44:28.447] Error starting mongod : [00:44:28.447] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41211,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41211 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 21652 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:44:27.271+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:44:27.271+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:44:27.271+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41211,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:44:27.273+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:44:27.274+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:44:27.276+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:44:27.349+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:44:27.349+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:44:27.360+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:44:27.410+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:44:27.410+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:44:27.947+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863467:947095][21652:0x7fcd18128bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:44:28.046+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ct... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:44:28.447+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:44:28.447] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:44:28.447+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:44:28.447] Initiating stopEphemeralMongo [2021-10-22T00:44:28.447+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:44:28.447] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:44:28.448+0000] [.error] [src/director/director.go:executePlan:962] [00:44:28.448] Failed to apply action. Result = : [00:44:28.447] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:44:28.447] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:41211 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:44:28.447] Error starting mongod : [00:44:28.447] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41211,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41211 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 21652 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:44:27.271+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:44:27.271+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:44:27.271+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41211,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:44:27.273+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:44:27.274+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:44:27.276+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:44:27.349+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:44:27.349+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:44:27.360+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:44:27.410+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:44:27.410+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:44:27.947+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863467:947095][21652:0x7fcd18128bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:44:28.448+0000] [.error] [src/director/director.go:planAndExecute:567] [00:44:28.448] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:44:28.448] Failed to apply action. Result = : [00:44:28.447] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:44:28.447] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:41211 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:44:28.447] Error starting mongod : [00:44:28.447] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41211,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41211 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 21652 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:44:27.271+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:44:27.271+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:44:27.271+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41211,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:44:27.273+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:44:27.274+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:44:27.276+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:44:27.349+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:44:27.349+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:44:27.360+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:44:27.410+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:44:27.410+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:44:27.947+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:44:28.448+0000] [.error] [src/director/director.go:mainLoop:399] [00:44:28.448] Failed to planAndExecute : [00:44:28.448] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:44:28.448] Failed to apply action. Result = : [00:44:28.447] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:44:28.447] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:41211 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:44:28.447] Error starting mongod : [00:44:28.447] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41211,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41211 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 21652 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:44:27.271+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:44:27.271+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:44:27.271+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41211,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:44:27.273+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:44:27.274+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:44:27.276+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:44:27.349+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:44:27.349+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:44:27.360+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:44:27.410+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:44:27.410+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:44:27.947+00:00"},"s":"I", "c":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:44:29.389+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:29.389] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:29.968+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:44:29.968] clusterConfig unchanged [2021-10-22T00:44:30.390+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:30.390] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:31.392+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:31.392] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:32.392+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:32.392] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:33.392+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:33.392] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:34.393+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:34.393] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:35.393+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:35.393] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:36.394+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:36.394] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:36.937+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:44:36.937] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.303594096s [2021-10-22T00:44:37.392+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:37.392] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:38.392+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:38.392] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:39.392+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:39.392] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:40.110+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:44:40.110] clusterConfig unchanged [2021-10-22T00:44:40.393+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:40.393] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:41.263+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:44:41.263] memory:rss=51105792 alloc=12868248 totalalloc=2155480208 mallocs=28127575 frees=28022999 heapinuse=16310272 heapobjects=104576 gcpausetotalns=1396630243 gcpauselastns=97913 gcnum=345 utimens=37120866000 stimens=10732604000 minflt=101327 majflt=405 [2021-10-22T00:44:41.399+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:41.399] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:42.394+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:42.394] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:43.395+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:43.395] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:44.138+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [00:44:44.138] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41211 (local=false) because it is not up [2021-10-22T00:44:44.395+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:44.395] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:45.396+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:45.396] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:46.397+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:46.397] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:47.403+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:47.403] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:48.399+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:48.399] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:49.399+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:49.399] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:50.243+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:44:50.243] clusterConfig unchanged [2021-10-22T00:44:50.399+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:50.399] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:51.400+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:51.400] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:52.400+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:52.400] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:53.401+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:53.401] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:54.402+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:54.402] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:55.402+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:55.402] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:56.403+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:56.403] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:57.403+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:57.403] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:58.409+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:58.409] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:44:59.405+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:44:59.405] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:00.344+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:45:00.344] clusterConfig unchanged [2021-10-22T00:45:00.409+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:00.409] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:01.406+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:01.406] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:02.407+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:02.407] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:03.407+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:03.407] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:04.411+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:04.411] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:05.409+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:05.409] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:06.410+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:06.410] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:07.411+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:07.411] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:08.411+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:08.411] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:09.412+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:09.412] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:10.418+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:10.417] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:10.446+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:45:10.446] clusterConfig unchanged [2021-10-22T00:45:11.412+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:11.412] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:12.413+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:12.413] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:13.413+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:13.413] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:14.413+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:14.413] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:15.416+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:15.416] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:16.414+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:16.414] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:17.415+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:17.415] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:18.415+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:18.415] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:19.416+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:19.416] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:19.612+0000] [.info] [src/director/director.go:computePlan:280] [00:45:19.612] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:45:19.612+0000] [.info] [src/director/director.go:tracef:794] [00:45:19.612] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:45:19.612+0000] [.info] [src/director/director.go:tracef:794] [00:45:19.612] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:45:19.612+0000] [.info] [src/director/director.go:tracef:794] [00:45:19.612] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:45:19.612+0000] [.info] [src/director/director.go:tracef:794] [00:45:19.612] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:45:19.612+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:45:19.612] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:45:19.774+0000] [.info] [src/director/director.go:tracef:794] [00:45:19.774] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:45:19.774+0000] [.info] [src/director/director.go:tracef:794] [00:45:19.774] because [2021-10-22T00:45:19.775+0000] [.info] [src/director/director.go:tracef:794] [00:45:19.775] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:45:19.775+0000] [.info] [src/director/director.go:tracef:794] [00:45:19.775] because ['currentState.CanBounceStop' = true] [2021-10-22T00:45:19.923+0000] [.info] [src/director/director.go:tracef:794] [00:45:19.923] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:45:19.923+0000] [.info] [src/director/director.go:tracef:794] [00:45:19.923] because [2021-10-22T00:45:19.923+0000] [.info] [src/director/director.go:tracef:794] [00:45:19.923] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:45:19.923+0000] [.info] [src/director/director.go:tracef:794] [00:45:19.923] because ['currentState.Up' = false] [2021-10-22T00:45:19.923+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:45:19.923] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:45:19.923+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:45:19.923] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:45:20.007+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:45:20.007] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:45:20.015+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:45:20.015] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:45:20.016+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:45:20.016] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:45:20.017+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:45:20.017] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:45:20.018+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:45:20.018] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:45:20.018+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:45:20.018] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:45:20.018+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:45:20.018] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:45:20.019+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:45:20.019] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:45:20.246+0000] [.info] [src/director/director.go:tracef:794] [00:45:20.246] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:45:20.246+0000] [.info] [src/director/director.go:tracef:794] [00:45:20.246] because ['currentState.Up' = false] [2021-10-22T00:45:20.420+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:20.420] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:20.474+0000] [.info] [src/director/director.go:tracef:794] [00:45:20.474] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:45:20.474+0000] [.info] [src/director/director.go:tracef:794] [00:45:20.474] because ['currentState.Up' = false] [2021-10-22T00:45:20.474+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:45:20.474] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:45:20.627+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:45:20.627] clusterConfig unchanged [2021-10-22T00:45:20.687+0000] [.info] [src/director/director.go:tracef:794] [00:45:20.687] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:45:20.688+0000] [.info] [src/director/director.go:tracef:794] [00:45:20.688] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:45:20.895+0000] [.info] [src/director/director.go:tracef:794] [00:45:20.895] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:45:20.895+0000] [.info] [src/director/director.go:tracef:794] [00:45:20.895] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:45:20.896+0000] [.info] [src/director/director.go:tracef:794] [00:45:20.896] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:45:20.896+0000] [.info] [src/director/director.go:tracef:794] [00:45:20.896] because ['currentState.Up' = false] [2021-10-22T00:45:20.896+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:45:20.896] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:45:20.896+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:45:20.896] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:45:20.928+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:45:20.928] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:45:20.929+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:45:20.929] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34457 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:45:21.426+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:21.426] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:21.658+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:45:21.658] ...Finished start [pid=40730] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34457,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:45:21.763+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:45:21.763] clusterConfig unchanged [2021-10-22T00:45:21.809+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:45:21.809] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 150.885545ms. [2021-10-22T00:45:21.809+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:45:21.809] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34457 (local=false) [2021-10-22T00:45:21.809+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:45:21.809] Dropping existing documents from system.replset [2021-10-22T00:45:21.938+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:45:21.938] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:45:21.942+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:45:21.942] Inserting oplogTruncateAfterPoint document [2021-10-22T00:45:21.942+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:45:21.942] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:45:21.943+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:45:21.943] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:45:21.943+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:45:21.943] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:45:21.956+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:45:21.956] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:45:21.956+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:45:21.956] Setting the minvalid document to [{_id ObjectID("617209a1523370002becc343")} {t -1} {ts {0 1}}] [2021-10-22T00:45:21.968+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:45:21.968] Inserted [{_id ObjectID("617209a1523370002becc343")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:45:21.968+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:45:21.968] Inserting new replica set configuration document. [2021-10-22T00:45:21.968+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:45:21.968] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:45:21.980+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:45:21.980] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:45:21.985+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:45:21.985] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:45:21.985+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:45:21.985] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:45:21.985+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:45:21.985] Initiating stopEphemeralMongo [2021-10-22T00:45:21.985+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:45:21.985] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34457 (local=false) [2021-10-22T00:45:21.985+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:45:21.985] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34457 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:45:22.353+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:45:22.353] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.516067ms. [2021-10-22T00:45:22.353+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:45:22.353] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34457 (local=false) [2021-10-22T00:45:22.353+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:45:22.353] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.169µs. [2021-10-22T00:45:22.353+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:45:22.353] Successfully finished running stopEphemeralMongo [2021-10-22T00:45:22.353+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:45:22.353] Successfully stopped existing ephemeral mongod. [2021-10-22T00:45:22.353+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:45:22.353] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:45:22.353+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:45:22.353] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:45:22.386+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:45:22.386] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:45:22.387+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:45:22.387] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60352 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:45:22.425+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:22.425] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:22.873+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:45:22.873] clusterConfig unchanged [2021-10-22T00:45:23.426+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:23.426] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:23.555+0000] [.error] [src/action/start.go:func1:93] [103] [00:45:23.555] Error starting mongod : [00:45:23.555] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60352,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60352 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:45:22.419Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 42290 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:45:22.439+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:45:22.439+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:45:22.439+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60352,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:45:22.441+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:45:22.444+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:45:22.466+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:45:22.500+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:45:22.500+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:45:22.500+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:45:22.544+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:45:22.544+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:45:23.051+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863523:51300][42290:0x7f838501cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:45:23.154+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863523:154073][42290:0x7f838501cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:45:23.253+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863523:253894][42290:0x7f838501cbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:45:23.369+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863523:369427][42290:0x7f838501cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:45:23.444+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863523:444707][42290:0x7f838501cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:45:23.504+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863523:504137][42290:0x7f838501cbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:45:23.504+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863523:504216][42290:0x7f838501cbc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:45:23.555+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:45:23.555] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60352 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:45:23.555] Error starting mongod : [00:45:23.555] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60352,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60352 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:45:22.419Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 42290 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:45:22.439+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:45:22.439+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:45:22.439+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60352,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:45:22.441+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:45:22.444+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:45:22.466+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:45:22.500+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:45:22.500+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:45:22.500+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:45:22.544+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:45:22.544+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:45:23.051+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863523:51300][42290:0x7f838501cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:45:23.154+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:45:23.555+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:45:23.555] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:45:23.555] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60352 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:45:23.555] Error starting mongod : [00:45:23.555] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60352,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60352 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:45:22.419Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 42290 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:45:22.439+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:45:22.439+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:45:22.439+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60352,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:45:22.441+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:45:22.444+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:45:22.466+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:45:22.500+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:45:22.500+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:45:22.500+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:45:22.544+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:45:22.544+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:45:23.051+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863523:51300][42290:0x7f838501cbc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:45:23.556+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:45:23.556] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:45:23.556+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:45:23.556] Initiating stopEphemeralMongo [2021-10-22T00:45:23.556+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:45:23.556] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:45:23.556+0000] [.error] [src/director/director.go:executePlan:962] [00:45:23.556] Failed to apply action. Result = : [00:45:23.555] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:45:23.555] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60352 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:45:23.555] Error starting mongod : [00:45:23.555] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60352,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60352 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:45:22.419Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 42290 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:45:22.439+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:45:22.439+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:45:22.439+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60352,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:45:22.441+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:45:22.444+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:45:22.466+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:45:22.500+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:45:22.500+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:45:22.500+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:45:22.544+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:45:22.544+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:45:23.051+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:45:23.556+0000] [.error] [src/director/director.go:planAndExecute:567] [00:45:23.556] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:45:23.556] Failed to apply action. Result = : [00:45:23.555] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:45:23.555] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60352 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:45:23.555] Error starting mongod : [00:45:23.555] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60352,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60352 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:45:22.419Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 42290 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:45:22.439+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:45:22.439+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:45:22.439+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60352,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:45:22.441+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:45:22.444+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:45:22.466+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:45:22.500+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:45:22.500+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:45:22.500+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:45:22.544+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:45:22.544+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:45:23.556+0000] [.error] [src/director/director.go:mainLoop:399] [00:45:23.556] Failed to planAndExecute : [00:45:23.556] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:45:23.556] Failed to apply action. Result = : [00:45:23.555] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:45:23.555] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60352 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:45:23.555] Error starting mongod : [00:45:23.555] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60352,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60352 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:45:22.419Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 42290 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:45:22.439+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:45:22.439+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:45:22.439+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60352,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:45:22.441+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:45:22.444+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:45:22.466+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:45:22.500+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:45:22.500+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:45:22.500+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:45:22.544+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:45:22.544+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:45:24.113+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:45:24.113] clusterConfig unchanged [2021-10-22T00:45:24.425+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:24.425] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:25.425+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:25.425] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:26.425+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:26.425] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:27.424+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:27.424] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:28.428+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:28.428] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:29.426+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:29.426] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:30.429+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:30.429] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:31.428+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:31.428] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:32.181+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:45:32.181] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.161861751s [2021-10-22T00:45:32.427+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:32.427] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:33.428+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:33.428] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:34.234+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:45:34.234] clusterConfig unchanged [2021-10-22T00:45:34.429+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:34.429] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:35.431+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:35.431] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:36.429+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:36.429] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:37.431+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:37.431] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:38.430+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:38.430] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:39.431+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:39.431] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:40.432+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:40.432] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:41.264+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:45:41.264] memory:rss=50085888 alloc=10760056 totalalloc=2240941024 mallocs=29198909 frees=29116865 heapinuse=14606336 heapobjects=82044 gcpausetotalns=1398519380 gcpauselastns=88906 gcnum=359 utimens=38555278000 stimens=11182714000 minflt=105602 majflt=405 [2021-10-22T00:45:41.432+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:41.432] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:42.435+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:42.435] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:43.434+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:43.434] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:44.405+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:45:44.405] clusterConfig unchanged [2021-10-22T00:45:44.435+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:44.435] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:45.435+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:45.435] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:46.436+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:46.436] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:47.437+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:47.437] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:48.438+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:48.438] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:49.438+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:49.438] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:50.438+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:50.438] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:51.438+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:51.438] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:52.439+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:52.439] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:53.443+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:53.443] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:54.440+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:54.440] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:54.526+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:45:54.526] clusterConfig unchanged [2021-10-22T00:45:55.441+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:55.441] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:56.441+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:56.441] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:57.441+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:57.441] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:58.442+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:58.442] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:45:59.442+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:45:59.442] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:00.443+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:00.443] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:01.443+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:01.443] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:02.444+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:02.444] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:03.449+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:03.449] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:04.450+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:04.450] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:04.678+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:46:04.678] clusterConfig unchanged [2021-10-22T00:46:05.445+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:05.445] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:06.446+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:06.446] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:07.447+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:07.447] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:08.447+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:08.447] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:09.448+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:09.448] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:10.450+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:10.450] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:11.449+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:11.449] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:12.449+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:12.449] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:13.450+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:13.450] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:14.450+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:14.450] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:14.748+0000] [.info] [src/director/director.go:computePlan:280] [00:46:14.748] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:46:14.748+0000] [.info] [src/director/director.go:tracef:794] [00:46:14.748] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:46:14.748+0000] [.info] [src/director/director.go:tracef:794] [00:46:14.748] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:46:14.748+0000] [.info] [src/director/director.go:tracef:794] [00:46:14.748] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:46:14.748+0000] [.info] [src/director/director.go:tracef:794] [00:46:14.748] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:46:14.748+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:46:14.748] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:46:14.770+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:46:14.770] clusterConfig unchanged [2021-10-22T00:46:14.901+0000] [.info] [src/director/director.go:tracef:794] [00:46:14.901] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:46:14.901+0000] [.info] [src/director/director.go:tracef:794] [00:46:14.901] because [2021-10-22T00:46:14.901+0000] [.info] [src/director/director.go:tracef:794] [00:46:14.901] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:46:14.901+0000] [.info] [src/director/director.go:tracef:794] [00:46:14.901] because ['currentState.CanBounceStop' = true] [2021-10-22T00:46:15.065+0000] [.info] [src/director/director.go:tracef:794] [00:46:15.065] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:46:15.065+0000] [.info] [src/director/director.go:tracef:794] [00:46:15.065] because [2021-10-22T00:46:15.065+0000] [.info] [src/director/director.go:tracef:794] [00:46:15.065] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:46:15.065+0000] [.info] [src/director/director.go:tracef:794] [00:46:15.065] because ['currentState.Up' = false] [2021-10-22T00:46:15.065+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:46:15.065] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:46:15.065+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:46:15.065] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:46:15.145+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:46:15.145] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:46:15.145+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:46:15.145] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:46:15.153+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:46:15.153] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:46:15.156+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:46:15.156] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:46:15.158+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:46:15.158] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:46:15.158+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:46:15.158] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:46:15.159+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:46:15.159] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:46:15.161+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:46:15.161] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:46:15.384+0000] [.info] [src/director/director.go:tracef:794] [00:46:15.384] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:46:15.384+0000] [.info] [src/director/director.go:tracef:794] [00:46:15.384] because ['currentState.Up' = false] [2021-10-22T00:46:15.472+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:15.472] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:15.597+0000] [.info] [src/director/director.go:tracef:794] [00:46:15.597] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:46:15.597+0000] [.info] [src/director/director.go:tracef:794] [00:46:15.597] because ['currentState.Up' = false] [2021-10-22T00:46:15.597+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:46:15.597] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:46:15.794+0000] [.info] [src/director/director.go:tracef:794] [00:46:15.794] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:46:15.794+0000] [.info] [src/director/director.go:tracef:794] [00:46:15.794] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:46:15.996+0000] [.info] [src/director/director.go:tracef:794] [00:46:15.996] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:46:15.996+0000] [.info] [src/director/director.go:tracef:794] [00:46:15.996] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:46:15.996+0000] [.info] [src/director/director.go:tracef:794] [00:46:15.996] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:46:15.996+0000] [.info] [src/director/director.go:tracef:794] [00:46:15.996] because ['currentState.Up' = false] [2021-10-22T00:46:15.996+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:46:15.996] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:46:15.996+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:46:15.996] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:46:16.033+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:46:16.033] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:46:16.033+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:46:16.033] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53327 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:46:16.461+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:16.461] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:16.753+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:46:16.753] ...Finished start [pid=61357] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53327,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:46:16.875+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:46:16.875] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 121.961918ms. [2021-10-22T00:46:16.875+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:46:16.875] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53327 (local=false) [2021-10-22T00:46:16.875+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:46:16.875] Dropping existing documents from system.replset [2021-10-22T00:46:16.997+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:46:16.997] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:46:17.000+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:46:17.000] Inserting oplogTruncateAfterPoint document [2021-10-22T00:46:17.000+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:46:17.000] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:46:17.002+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:46:17.002] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:46:17.002+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:46:17.002] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:46:17.018+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:46:17.018] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:46:17.018+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:46:17.018] Setting the minvalid document to [{_id ObjectID("617209d9523370002becc351")} {t -1} {ts {0 1}}] [2021-10-22T00:46:17.030+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:46:17.030] Inserted [{_id ObjectID("617209d9523370002becc351")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:46:17.030+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:46:17.030] Inserting new replica set configuration document. [2021-10-22T00:46:17.030+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:46:17.030] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:46:17.043+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:46:17.043] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:46:17.048+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:46:17.048] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:46:17.048+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:46:17.048] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:46:17.048+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:46:17.048] Initiating stopEphemeralMongo [2021-10-22T00:46:17.048+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:46:17.048] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53327 (local=false) [2021-10-22T00:46:17.048+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:46:17.048] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53327 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:46:17.397+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:46:17.397] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.368652ms. [2021-10-22T00:46:17.397+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:46:17.397] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53327 (local=false) [2021-10-22T00:46:17.397+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:46:17.397] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.412µs. [2021-10-22T00:46:17.397+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:46:17.397] Successfully finished running stopEphemeralMongo [2021-10-22T00:46:17.397+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:46:17.397] Successfully stopped existing ephemeral mongod. [2021-10-22T00:46:17.397+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:46:17.397] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:46:17.397+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:46:17.397] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:46:17.435+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:46:17.435] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:46:17.436+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:46:17.436] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51290 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:46:17.460+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:17.460] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:18.463+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:18.463] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:18.555+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [00:46:18.555] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T00:46:18.563+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:18.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T00:46:18.563+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [00:46:18.563] Could not get version : [00:46:18.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T00:46:18.563+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [00:46:18.563] Error getting version for log lines. Skipping : [00:46:18.563] Could not get version : [00:46:18.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T00:46:18.592+0000] [.error] [src/action/start.go:func1:93] [103] [00:46:18.592] Error starting mongod : [00:46:18.592] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51290,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51290 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 62861 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:46:17.486+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:46:17.486+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:46:17.486+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51290,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:46:17.488+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:46:17.489+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:46:17.490+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:46:17.534+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:46:17.534+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:46:17.534+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:46:17.579+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:46:17.579+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:46:18.092+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863578:92823][62861:0x7f2d02e0abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:46:18.195+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863578:195556][62861:0x7f2d02e0abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:46:18.294+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863578:294253][62861:0x7f2d02e0abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T00:46:18.405+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863578:405589][62861:0x7f2d02e0abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:46:18.477+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863578:477700][62861:0x7f2d02e0abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:46:18.534+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863578:534444][62861:0x7f2d02e0abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:46:18.534+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863578:534519][62861:0x7f2d02e0abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:46:18.535+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:46:18.592+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:46:18.592] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51290 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:46:18.592] Error starting mongod : [00:46:18.592] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51290,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51290 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 62861 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:46:17.486+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:46:17.486+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:46:17.486+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51290,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:46:17.488+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:46:17.489+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:46:17.490+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:46:17.534+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:46:17.534+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:46:17.534+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:46:17.579+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:46:17.579+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:46:18.092+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863578:92823][62861:0x7f2d02e0abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:46:18.195+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863578:195556][62861:0x7f2d02e0abc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:46:18.592+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:46:18.592] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:46:18.592] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51290 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:46:18.592] Error starting mongod : [00:46:18.592] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51290,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51290 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 62861 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:46:17.486+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:46:17.486+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:46:17.486+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51290,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:46:17.488+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:46:17.489+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:46:17.490+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:46:17.534+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:46:17.534+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:46:17.534+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:46:17.579+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:46:17.579+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:46:18.092+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863578:92823][62861:0x7f2d02e0abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:46:18.195+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:46:18.592+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:46:18.592] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:46:18.592+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:46:18.592] Initiating stopEphemeralMongo [2021-10-22T00:46:18.593+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:46:18.592] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:46:18.593+0000] [.error] [src/director/director.go:executePlan:962] [00:46:18.593] Failed to apply action. Result = : [00:46:18.592] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:46:18.592] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51290 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:46:18.592] Error starting mongod : [00:46:18.592] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51290,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51290 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 62861 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:46:17.486+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:46:17.486+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:46:17.486+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51290,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:46:17.488+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:46:17.489+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:46:17.490+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:46:17.534+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:46:17.534+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:46:17.534+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:46:17.579+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:46:17.579+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:46:18.092+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863578:92823][62861:0x7f2d02e0abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:46:18.593+0000] [.error] [src/director/director.go:planAndExecute:567] [00:46:18.593] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:46:18.593] Failed to apply action. Result = : [00:46:18.592] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:46:18.592] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51290 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:46:18.592] Error starting mongod : [00:46:18.592] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51290,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51290 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 62861 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:46:17.486+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:46:17.486+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:46:17.486+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51290,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:46:17.488+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:46:17.489+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:46:17.490+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:46:17.534+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:46:17.534+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:46:17.534+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:46:17.579+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:46:17.579+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:46:18.092+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:46:18.593+0000] [.error] [src/director/director.go:mainLoop:399] [00:46:18.593] Failed to planAndExecute : [00:46:18.593] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:46:18.593] Failed to apply action. Result = : [00:46:18.592] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:46:18.592] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51290 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:46:18.592] Error starting mongod : [00:46:18.592] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51290,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51290 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 62861 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:46:17.486+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:46:17.486+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:46:17.486+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51290,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:46:17.488+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:46:17.489+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:46:17.490+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:46:17.534+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:46:17.534+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:46:17.534+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:46:17.579+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:46:17.579+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:46:18.092+00:00"},"s":"I", "c":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:46:19.460+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:19.460] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:20.463+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:20.462] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:21.463+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:21.463] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:22.462+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:22.462] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:23.463+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:23.463] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:24.463+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:24.463] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:25.010+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:46:25.010] clusterConfig unchanged [2021-10-22T00:46:25.466+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:25.466] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:26.465+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:26.465] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:27.301+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:46:27.301] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.139459265s [2021-10-22T00:46:27.463+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:27.463] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:28.464+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:28.464] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:29.465+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:29.465] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:30.466+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:30.466] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:31.483+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:31.483] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:32.468+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:32.468] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:33.468+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:33.468] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:34.469+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:34.469] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:35.122+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:46:35.122] clusterConfig unchanged [2021-10-22T00:46:35.471+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:35.471] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:36.470+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:36.470] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:37.476+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:37.476] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:38.471+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:38.471] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:39.471+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:39.471] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:40.472+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:40.472] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:41.264+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:46:41.264] memory:rss=51326976 alloc=8496240 totalalloc=2326223760 mallocs=30259693 frees=30201004 heapinuse=13975552 heapobjects=58689 gcpausetotalns=1401415795 gcpauselastns=102315 gcnum=373 utimens=40018022000 stimens=11657729000 minflt=110139 majflt=405 [2021-10-22T00:46:41.473+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:41.473] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:42.476+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:42.476] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:43.474+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:43.474] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:44.474+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:44.474] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:45.275+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:46:45.275] clusterConfig unchanged [2021-10-22T00:46:45.474+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:45.474] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:46.479+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:46.479] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:47.475+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:47.475] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:48.480+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:48.480] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:49.476+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:49.476] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:50.477+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:50.477] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:51.477+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:51.477] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:52.478+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:52.478] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:53.478+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:53.478] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:54.479+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:54.479] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:55.406+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:46:55.406] clusterConfig unchanged [2021-10-22T00:46:55.480+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:55.480] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:56.481+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:56.481] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:57.481+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:57.481] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:58.482+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:58.482] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:46:59.485+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:46:59.485] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:00.484+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:00.483] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:01.484+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:01.484] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:02.484+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:02.484] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:03.485+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:03.485] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:04.486+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:04.486] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:05.486+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:05.486] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:05.538+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:47:05.538] clusterConfig unchanged [2021-10-22T00:47:06.487+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:06.487] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:07.488+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:07.488] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:08.489+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:08.489] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:09.489+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:09.489] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:10.489+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:10.489] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:11.490+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:11.490] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:12.491+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:12.491] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:13.493+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:13.493] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:14.491+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:14.491] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:14.766+0000] [.info] [src/director/director.go:computePlan:280] [00:47:14.765] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:47:14.766+0000] [.info] [src/director/director.go:tracef:794] [00:47:14.766] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:47:14.766+0000] [.info] [src/director/director.go:tracef:794] [00:47:14.766] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:47:14.767+0000] [.info] [src/director/director.go:tracef:794] [00:47:14.767] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:47:14.767+0000] [.info] [src/director/director.go:tracef:794] [00:47:14.767] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:47:14.767+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:47:14.767] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:47:14.945+0000] [.info] [src/director/director.go:tracef:794] [00:47:14.945] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:47:14.945+0000] [.info] [src/director/director.go:tracef:794] [00:47:14.945] because [2021-10-22T00:47:14.945+0000] [.info] [src/director/director.go:tracef:794] [00:47:14.945] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:47:14.945+0000] [.info] [src/director/director.go:tracef:794] [00:47:14.945] because ['currentState.CanBounceStop' = true] [2021-10-22T00:47:15.111+0000] [.info] [src/director/director.go:tracef:794] [00:47:15.111] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:47:15.111+0000] [.info] [src/director/director.go:tracef:794] [00:47:15.111] because [2021-10-22T00:47:15.112+0000] [.info] [src/director/director.go:tracef:794] [00:47:15.112] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:47:15.112+0000] [.info] [src/director/director.go:tracef:794] [00:47:15.112] because ['currentState.Up' = false] [2021-10-22T00:47:15.112+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:47:15.112] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:47:15.112+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:47:15.112] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:47:15.197+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:47:15.197] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:47:15.205+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:47:15.205] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:47:15.206+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:47:15.206] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:47:15.207+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:47:15.207] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:47:15.207+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:47:15.207] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:47:15.208+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:47:15.208] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:47:15.208+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:47:15.208] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:47:15.209+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:47:15.209] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:47:15.425+0000] [.info] [src/director/director.go:tracef:794] [00:47:15.425] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:47:15.425+0000] [.info] [src/director/director.go:tracef:794] [00:47:15.425] because ['currentState.Up' = false] [2021-10-22T00:47:15.514+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:15.514] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:15.640+0000] [.info] [src/director/director.go:tracef:794] [00:47:15.640] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:47:15.641+0000] [.info] [src/director/director.go:tracef:794] [00:47:15.641] because ['currentState.Up' = false] [2021-10-22T00:47:15.641+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:47:15.641] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:47:15.763+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:47:15.763] clusterConfig unchanged [2021-10-22T00:47:15.851+0000] [.info] [src/director/director.go:tracef:794] [00:47:15.851] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:47:15.851+0000] [.info] [src/director/director.go:tracef:794] [00:47:15.851] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:47:16.046+0000] [.info] [src/director/director.go:tracef:794] [00:47:16.046] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:47:16.046+0000] [.info] [src/director/director.go:tracef:794] [00:47:16.046] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:47:16.047+0000] [.info] [src/director/director.go:tracef:794] [00:47:16.047] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:47:16.047+0000] [.info] [src/director/director.go:tracef:794] [00:47:16.047] because ['currentState.Up' = false] [2021-10-22T00:47:16.047+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:47:16.047] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:47:16.047+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:47:16.047] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:47:16.079+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:47:16.079] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:47:16.080+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:47:16.080] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39052 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:47:16.503+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:16.503] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:16.818+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:47:16.818] ...Finished start [pid=8015] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39052,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:47:16.877+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:47:16.877] clusterConfig unchanged [2021-10-22T00:47:16.924+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:47:16.924] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 105.909396ms. [2021-10-22T00:47:16.924+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:47:16.924] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39052 (local=false) [2021-10-22T00:47:16.924+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:47:16.924] Dropping existing documents from system.replset [2021-10-22T00:47:17.056+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:47:17.056] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:47:17.060+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:47:17.060] Inserting oplogTruncateAfterPoint document [2021-10-22T00:47:17.060+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:47:17.060] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:47:17.061+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:47:17.061] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:47:17.061+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:47:17.061] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:47:17.074+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:47:17.074] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:47:17.074+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:47:17.074] Setting the minvalid document to [{_id ObjectID("61720a15523370002becc35f")} {t -1} {ts {0 1}}] [2021-10-22T00:47:17.088+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:47:17.088] Inserted [{_id ObjectID("61720a15523370002becc35f")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:47:17.088+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:47:17.088] Inserting new replica set configuration document. [2021-10-22T00:47:17.088+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:47:17.088] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:47:17.100+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:47:17.100] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:47:17.104+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:47:17.104] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:47:17.104+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:47:17.104] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:47:17.104+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:47:17.104] Initiating stopEphemeralMongo [2021-10-22T00:47:17.104+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:47:17.104] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39052 (local=false) [2021-10-22T00:47:17.104+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:47:17.104] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39052 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:47:17.426+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:47:17.425] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.998192ms. [2021-10-22T00:47:17.426+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:47:17.426] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39052 (local=false) [2021-10-22T00:47:17.426+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:47:17.426] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 44.485µs. [2021-10-22T00:47:17.426+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:47:17.426] Successfully finished running stopEphemeralMongo [2021-10-22T00:47:17.426+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:47:17.426] Successfully stopped existing ephemeral mongod. [2021-10-22T00:47:17.426+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:47:17.426] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:47:17.426+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:47:17.426] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:47:17.461+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:47:17.461] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:47:17.461+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:47:17.461] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34396 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:47:17.512+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:17.512] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:17.975+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:47:17.975] clusterConfig unchanged [2021-10-22T00:47:18.514+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:18.514] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:18.691+0000] [.error] [src/action/start.go:func1:93] [103] [00:47:18.691] Error starting mongod : [00:47:18.690] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34396,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34396 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:47:17.493Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 9482 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:47:17.522+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:47:17.522+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:47:17.522+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34396,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:47:17.523+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:47:17.526+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:47:17.527+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:47:17.577+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:47:17.577+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:47:17.577+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:47:17.634+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:47:17.634+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:47:18.180+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863638:180415][9482:0x7fea96ae5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:47:18.281+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863638:281746][9482:0x7fea96ae5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:47:18.382+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863638:382085][9482:0x7fea96ae5bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:47:18.493+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863638:493841][9482:0x7fea96ae5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:47:18.565+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863638:565454][9482:0x7fea96ae5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:47:18.637+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863638:637258][9482:0x7fea96ae5bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:47:18.637+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863638:637327][9482:0x7fea96ae5bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:47:18.691+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:47:18.691] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34396 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:47:18.691] Error starting mongod : [00:47:18.690] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34396,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34396 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:47:17.493Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 9482 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:47:17.522+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:47:17.522+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:47:17.522+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34396,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:47:17.523+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:47:17.526+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:47:17.527+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:47:17.577+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:47:17.577+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:47:17.577+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:47:17.634+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:47:17.634+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:47:18.180+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863638:180415][9482:0x7fea96ae5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:47:18.281+00:00"},"s...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:47:18.691+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:47:18.691] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:47:18.691] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34396 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:47:18.691] Error starting mongod : [00:47:18.690] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34396,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34396 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:47:17.493Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 9482 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:47:17.522+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:47:17.522+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:47:17.522+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34396,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:47:17.523+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:47:17.526+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:47:17.527+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:47:17.577+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:47:17.577+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:47:17.577+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:47:17.634+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:47:17.634+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:47:18.180+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863638:180415][9482:0x7fea96ae5bc0... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:47:18.691+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:47:18.691] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:47:18.691+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:47:18.691] Initiating stopEphemeralMongo [2021-10-22T00:47:18.691+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:47:18.691] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:47:18.692+0000] [.error] [src/director/director.go:executePlan:962] [00:47:18.692] Failed to apply action. Result = : [00:47:18.691] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:47:18.691] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34396 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:47:18.691] Error starting mongod : [00:47:18.690] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34396,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34396 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:47:17.493Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 9482 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:47:17.522+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:47:17.522+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:47:17.522+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34396,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:47:17.523+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:47:17.526+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:47:17.527+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:47:17.577+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:47:17.577+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:47:17.577+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:47:17.634+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:47:17.634+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:47:18.180+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:47:18.692+0000] [.error] [src/director/director.go:planAndExecute:567] [00:47:18.692] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:47:18.692] Failed to apply action. Result = : [00:47:18.691] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:47:18.691] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34396 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:47:18.691] Error starting mongod : [00:47:18.690] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34396,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34396 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:47:17.493Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 9482 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:47:17.522+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:47:17.522+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:47:17.522+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34396,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:47:17.523+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:47:17.526+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:47:17.527+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:47:17.577+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:47:17.577+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:47:17.577+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:47:17.634+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:47:17.634+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:47:18.692+0000] [.error] [src/director/director.go:mainLoop:399] [00:47:18.692] Failed to planAndExecute : [00:47:18.692] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:47:18.692] Failed to apply action. Result = : [00:47:18.691] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:47:18.691] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34396 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:47:18.691] Error starting mongod : [00:47:18.690] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34396,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34396 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:47:17.493Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 9482 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:47:17.522+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:47:17.522+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:47:17.522+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34396,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:47:17.523+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:47:17.526+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:47:17.527+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:47:17.577+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:47:17.577+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:47:17.577+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:47:17.634+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:47:17.634+00:00"},"s":"I", "c":"NETWORK", "id"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:47:19.253+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:47:19.253] clusterConfig unchanged [2021-10-22T00:47:19.513+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:19.513] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:20.513+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:20.513] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:21.515+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:21.515] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:22.516+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:22.516] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:23.515+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:23.515] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:24.515+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:24.515] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:25.516+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:25.516] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:26.519+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:26.519] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:27.435+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:47:27.435] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.225969453s [2021-10-22T00:47:27.516+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:27.516] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:27.557+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [00:47:27.557] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T00:47:27.560+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [00:47:27.560] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T00:47:28.517+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:28.517] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:29.385+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:47:29.385] clusterConfig unchanged [2021-10-22T00:47:29.518+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:29.518] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:30.537+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:30.537] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:31.520+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:31.520] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:32.521+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:32.520] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:33.521+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:33.521] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:34.523+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [00:47:34.523] lsblk cache invalidated, running lsblk command [2021-10-22T00:47:34.524+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:34.524] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:35.522+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:35.522] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:36.530+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:36.530] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:37.523+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:37.523] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:38.528+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:38.528] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:39.517+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:47:39.517] clusterConfig unchanged [2021-10-22T00:47:39.524+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:39.524] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:40.525+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:40.525] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:41.265+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:47:41.265] memory:rss=51097600 alloc=7967080 totalalloc=2411339896 mallocs=31332406 frees=31278989 heapinuse=13795328 heapobjects=53417 gcpausetotalns=1403237588 gcpauselastns=114777 gcnum=387 utimens=41489887000 stimens=12156413000 minflt=114391 majflt=405 [2021-10-22T00:47:41.525+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:41.525] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:42.525+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:42.525] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:43.525+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:43.525] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:44.147+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [00:47:44.147] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34396 (local=false) because it is not up [2021-10-22T00:47:44.525+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:44.525] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:45.526+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:45.526] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:46.526+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:46.526] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:47.526+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:47.526] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:48.528+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:48.528] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:49.527+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:49.527] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:49.649+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:47:49.649] clusterConfig unchanged [2021-10-22T00:47:50.528+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:50.528] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:51.529+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:51.529] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:52.530+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:52.530] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:53.531+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:53.530] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:54.531+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:54.531] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:55.532+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:55.532] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:56.532+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:56.532] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:57.533+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:57.533] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:58.533+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:58.533] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:59.533+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:47:59.533] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:47:59.771+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:47:59.771] clusterConfig unchanged [2021-10-22T00:48:00.534+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:00.534] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:01.536+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:01.535] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:02.536+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:02.536] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:03.540+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:03.540] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:04.538+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:04.538] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:05.539+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:05.539] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:06.551+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:06.551] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:07.555+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:07.554] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:08.540+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:08.540] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:09.540+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:09.540] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:09.873+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:48:09.873] clusterConfig unchanged [2021-10-22T00:48:10.541+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:10.541] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:11.542+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:11.542] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:12.543+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:12.543] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:13.543+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:13.543] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:14.544+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:14.543] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:14.855+0000] [.info] [src/director/director.go:computePlan:280] [00:48:14.855] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:48:14.855+0000] [.info] [src/director/director.go:tracef:794] [00:48:14.855] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:48:14.855+0000] [.info] [src/director/director.go:tracef:794] [00:48:14.855] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:48:14.855+0000] [.info] [src/director/director.go:tracef:794] [00:48:14.855] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:48:14.855+0000] [.info] [src/director/director.go:tracef:794] [00:48:14.855] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:48:14.856+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:48:14.856] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:48:15.007+0000] [.info] [src/director/director.go:tracef:794] [00:48:15.007] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:48:15.007+0000] [.info] [src/director/director.go:tracef:794] [00:48:15.007] because [2021-10-22T00:48:15.007+0000] [.info] [src/director/director.go:tracef:794] [00:48:15.007] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:48:15.007+0000] [.info] [src/director/director.go:tracef:794] [00:48:15.007] because ['currentState.CanBounceStop' = true] [2021-10-22T00:48:15.158+0000] [.info] [src/director/director.go:tracef:794] [00:48:15.158] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:48:15.158+0000] [.info] [src/director/director.go:tracef:794] [00:48:15.158] because [2021-10-22T00:48:15.158+0000] [.info] [src/director/director.go:tracef:794] [00:48:15.158] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:48:15.158+0000] [.info] [src/director/director.go:tracef:794] [00:48:15.158] because ['currentState.Up' = false] [2021-10-22T00:48:15.158+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:48:15.158] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:48:15.158+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:48:15.158] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:48:15.239+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:48:15.239] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:48:15.247+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:48:15.247] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:48:15.248+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:48:15.248] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:48:15.248+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:48:15.248] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:48:15.249+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:48:15.249] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:48:15.250+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:48:15.250] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:48:15.250+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:48:15.250] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:48:15.251+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:48:15.251] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:48:15.469+0000] [.info] [src/director/director.go:tracef:794] [00:48:15.469] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:48:15.469+0000] [.info] [src/director/director.go:tracef:794] [00:48:15.469] because ['currentState.Up' = false] [2021-10-22T00:48:15.561+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:15.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:15.693+0000] [.info] [src/director/director.go:tracef:794] [00:48:15.693] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:48:15.693+0000] [.info] [src/director/director.go:tracef:794] [00:48:15.693] because ['currentState.Up' = false] [2021-10-22T00:48:15.693+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:48:15.693] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:48:15.917+0000] [.info] [src/director/director.go:tracef:794] [00:48:15.917] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:48:15.917+0000] [.info] [src/director/director.go:tracef:794] [00:48:15.917] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:48:16.118+0000] [.info] [src/director/director.go:tracef:794] [00:48:16.118] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:48:16.118+0000] [.info] [src/director/director.go:tracef:794] [00:48:16.118] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:48:16.119+0000] [.info] [src/director/director.go:tracef:794] [00:48:16.118] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:48:16.119+0000] [.info] [src/director/director.go:tracef:794] [00:48:16.119] because ['currentState.Up' = false] [2021-10-22T00:48:16.119+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:48:16.119] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:48:16.119+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:48:16.119] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:48:16.153+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:48:16.153] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:48:16.154+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:48:16.153] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38695 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:48:16.548+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:16.548] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:16.868+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:48:16.868] ...Finished start [pid=28895] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38695,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:48:16.995+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:48:16.995] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 126.068011ms. [2021-10-22T00:48:16.995+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:48:16.995] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38695 (local=false) [2021-10-22T00:48:16.995+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:48:16.995] Dropping existing documents from system.replset [2021-10-22T00:48:17.238+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:48:17.238] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:48:17.242+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:48:17.242] Inserting oplogTruncateAfterPoint document [2021-10-22T00:48:17.242+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:48:17.242] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:48:17.243+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:48:17.243] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:48:17.243+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:48:17.243] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:48:17.258+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:48:17.258] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:48:17.258+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:48:17.258] Setting the minvalid document to [{_id ObjectID("61720a51523370002becc36d")} {t -1} {ts {0 1}}] [2021-10-22T00:48:17.272+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:48:17.272] Inserted [{_id ObjectID("61720a51523370002becc36d")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:48:17.272+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:48:17.272] Inserting new replica set configuration document. [2021-10-22T00:48:17.272+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:48:17.272] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:48:17.285+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:48:17.285] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:48:17.288+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:48:17.288] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:48:17.288+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:48:17.288] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:48:17.288+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:48:17.288] Initiating stopEphemeralMongo [2021-10-22T00:48:17.288+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:48:17.288] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38695 (local=false) [2021-10-22T00:48:17.288+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:48:17.288] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38695 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:48:17.557+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:17.557] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:17.733+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:48:17.733] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.129782ms. [2021-10-22T00:48:17.733+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:48:17.733] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38695 (local=false) [2021-10-22T00:48:17.733+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:48:17.733] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.949µs. [2021-10-22T00:48:17.733+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:48:17.733] Successfully finished running stopEphemeralMongo [2021-10-22T00:48:17.733+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:48:17.733] Successfully stopped existing ephemeral mongod. [2021-10-22T00:48:17.733+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:48:17.733] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:48:17.733+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:48:17.733] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:48:17.771+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:48:17.771] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:48:17.771+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:48:17.771] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55386 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:48:18.558+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:18.558] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:19.015+0000] [.error] [src/action/start.go:func1:93] [103] [00:48:19.015] Error starting mongod : [00:48:19.014] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55386,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55386 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:48:17.801Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 30600 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:48:17.855+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:48:17.855+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:48:17.855+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55386,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:48:17.857+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:48:17.865+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:48:17.866+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:48:17.882+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:48:17.883+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:48:17.883+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:48:17.928+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:48:17.929+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:48:18.503+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863698:503563][30600:0x7f3474bc8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:48:18.606+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863698:606634][30600:0x7f3474bc8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:48:18.709+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863698:709559][30600:0x7f3474bc8bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T00:48:18.821+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863698:821227][30600:0x7f3474bc8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:48:18.895+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863698:895011][30600:0x7f3474bc8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:48:18.956+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863698:956124][30600:0x7f3474bc8bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:48:18.956+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863698:956201][30600:0x7f3474bc8bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:48:19.015+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:48:19.015] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:55386 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:48:19.015] Error starting mongod : [00:48:19.014] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55386,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55386 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:48:17.801Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 30600 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:48:17.855+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:48:17.855+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:48:17.855+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55386,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:48:17.857+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:48:17.865+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:48:17.866+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:48:17.882+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:48:17.883+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:48:17.883+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:48:17.928+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:48:17.929+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:48:18.503+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863698:503563][30600:0x7f3474bc8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:48:18.606+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:48:19.015+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:48:19.015] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:48:19.015] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:55386 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:48:19.015] Error starting mongod : [00:48:19.014] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55386,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55386 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:48:17.801Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 30600 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:48:17.855+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:48:17.855+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:48:17.855+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55386,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:48:17.857+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:48:17.865+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:48:17.866+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:48:17.882+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:48:17.883+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:48:17.883+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:48:17.928+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:48:17.929+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:48:18.503+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863698:503563][30600:0x7f3474bc8bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:48:19.015+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:48:19.015] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:48:19.015+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:48:19.015] Initiating stopEphemeralMongo [2021-10-22T00:48:19.015+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:48:19.015] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:48:19.016+0000] [.error] [src/director/director.go:executePlan:962] [00:48:19.016] Failed to apply action. Result = : [00:48:19.015] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:48:19.015] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:55386 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:48:19.015] Error starting mongod : [00:48:19.014] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55386,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55386 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:48:17.801Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 30600 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:48:17.855+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:48:17.855+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:48:17.855+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55386,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:48:17.857+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:48:17.865+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:48:17.866+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:48:17.882+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:48:17.883+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:48:17.883+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:48:17.928+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:48:17.929+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:48:18.503+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:48:19.016+0000] [.error] [src/director/director.go:planAndExecute:567] [00:48:19.016] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:48:19.016] Failed to apply action. Result = : [00:48:19.015] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:48:19.015] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:55386 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:48:19.015] Error starting mongod : [00:48:19.014] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55386,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55386 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:48:17.801Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 30600 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:48:17.855+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:48:17.855+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:48:17.855+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55386,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:48:17.857+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:48:17.865+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:48:17.866+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:48:17.882+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:48:17.883+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:48:17.883+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:48:17.928+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:48:17.929+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:48:19.016+0000] [.error] [src/director/director.go:mainLoop:399] [00:48:19.016] Failed to planAndExecute : [00:48:19.016] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:48:19.016] Failed to apply action. Result = : [00:48:19.015] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:48:19.015] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:55386 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:48:19.015] Error starting mongod : [00:48:19.014] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55386,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55386 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:48:17.801Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 30600 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:48:17.855+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:48:17.855+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:48:17.855+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55386,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:48:17.857+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:48:17.865+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:48:17.866+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:48:17.882+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:48:17.883+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:48:17.883+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:48:17.928+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:48:17.929+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:48:19.558+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:19.558] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:20.113+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:48:20.113] clusterConfig unchanged [2021-10-22T00:48:20.559+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:20.559] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:21.560+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:21.559] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:22.560+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:22.560] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:23.560+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:23.560] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:24.559+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:24.559] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:25.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:25.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:26.563+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:26.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:27.509+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:48:27.509] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.258312076s [2021-10-22T00:48:27.563+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:27.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:28.563+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:28.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:29.565+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:29.565] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:30.275+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:48:30.275] clusterConfig unchanged [2021-10-22T00:48:30.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:30.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:31.575+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:31.574] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:32.564+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:32.564] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:33.567+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:33.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:34.567+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:34.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:35.566+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:35.566] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:36.569+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:36.569] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:37.568+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:37.568] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:38.569+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:38.569] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:39.569+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:39.569] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:40.416+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:48:40.416] clusterConfig unchanged [2021-10-22T00:48:40.570+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:40.570] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:41.265+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:48:41.265] memory:rss=51372032 alloc=11619304 totalalloc=2496346888 mallocs=32396776 frees=32304664 heapinuse=15360000 heapobjects=92112 gcpausetotalns=1405722184 gcpauselastns=103932 gcnum=400 utimens=43125512000 stimens=12789386000 minflt=118756 majflt=405 [2021-10-22T00:48:41.571+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:41.571] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:42.571+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:42.571] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:43.571+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:43.571] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:44.572+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:44.572] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:45.573+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:45.573] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:46.576+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:46.576] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:47.573+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:47.573] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:48.573+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:48.573] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:49.578+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:49.578] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:50.536+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:48:50.536] clusterConfig unchanged [2021-10-22T00:48:50.575+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:50.575] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:51.575+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:51.575] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:52.576+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:52.576] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:53.577+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:53.577] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:54.578+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:54.578] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:55.578+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:55.578] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:56.578+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:56.578] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:57.578+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:57.578] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:58.579+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:58.579] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:48:59.579+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:48:59.579] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:00.580+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:00.580] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:00.648+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:49:00.648] clusterConfig unchanged [2021-10-22T00:49:01.581+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:01.581] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:02.581+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:02.581] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:03.591+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:03.591] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:04.582+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:04.582] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:05.583+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:05.583] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:06.583+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:06.583] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:07.585+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:07.585] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:08.583+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:08.583] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:09.584+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:09.584] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:10.585+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:10.585] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:10.829+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:49:10.829] clusterConfig unchanged [2021-10-22T00:49:11.586+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:11.586] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:12.586+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:12.586] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:13.587+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:13.587] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:14.593+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:14.593] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:15.588+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:15.588] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:16.588+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:16.588] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:17.589+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:17.589] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:18.589+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:18.589] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:19.192+0000] [.info] [src/director/director.go:computePlan:280] [00:49:19.192] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:49:19.193+0000] [.info] [src/director/director.go:tracef:794] [00:49:19.193] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:49:19.193+0000] [.info] [src/director/director.go:tracef:794] [00:49:19.193] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:49:19.193+0000] [.info] [src/director/director.go:tracef:794] [00:49:19.193] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:49:19.193+0000] [.info] [src/director/director.go:tracef:794] [00:49:19.193] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:49:19.193+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:49:19.193] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:49:19.351+0000] [.info] [src/director/director.go:tracef:794] [00:49:19.351] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:49:19.351+0000] [.info] [src/director/director.go:tracef:794] [00:49:19.351] because [2021-10-22T00:49:19.351+0000] [.info] [src/director/director.go:tracef:794] [00:49:19.351] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:49:19.352+0000] [.info] [src/director/director.go:tracef:794] [00:49:19.351] because ['currentState.CanBounceStop' = true] [2021-10-22T00:49:19.499+0000] [.info] [src/director/director.go:tracef:794] [00:49:19.499] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:49:19.500+0000] [.info] [src/director/director.go:tracef:794] [00:49:19.500] because [2021-10-22T00:49:19.500+0000] [.info] [src/director/director.go:tracef:794] [00:49:19.500] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:49:19.500+0000] [.info] [src/director/director.go:tracef:794] [00:49:19.500] because ['currentState.Up' = false] [2021-10-22T00:49:19.500+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:49:19.500] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:49:19.500+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:49:19.500] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:49:19.583+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:49:19.583] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:49:19.593+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:19.593] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:19.593+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:49:19.593] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:49:19.594+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:49:19.594] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:49:19.594+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:49:19.594] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:49:19.596+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:49:19.596] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:49:19.596+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:49:19.596] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:49:19.596+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:49:19.596] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:49:19.598+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:49:19.598] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:49:19.805+0000] [.info] [src/director/director.go:tracef:794] [00:49:19.804] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:49:19.805+0000] [.info] [src/director/director.go:tracef:794] [00:49:19.805] because ['currentState.Up' = false] [2021-10-22T00:49:20.003+0000] [.info] [src/director/director.go:tracef:794] [00:49:20.003] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:49:20.003+0000] [.info] [src/director/director.go:tracef:794] [00:49:20.003] because ['currentState.Up' = false] [2021-10-22T00:49:20.003+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:49:20.003] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:49:20.200+0000] [.info] [src/director/director.go:tracef:794] [00:49:20.200] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:49:20.200+0000] [.info] [src/director/director.go:tracef:794] [00:49:20.200] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:49:20.422+0000] [.info] [src/director/director.go:tracef:794] [00:49:20.422] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:49:20.422+0000] [.info] [src/director/director.go:tracef:794] [00:49:20.422] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:49:20.422+0000] [.info] [src/director/director.go:tracef:794] [00:49:20.422] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:49:20.422+0000] [.info] [src/director/director.go:tracef:794] [00:49:20.422] because ['currentState.Up' = false] [2021-10-22T00:49:20.422+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:49:20.422] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:49:20.422+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:49:20.422] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:49:20.456+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:49:20.456] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:49:20.457+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:49:20.457] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38508 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:49:20.595+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:20.595] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:21.069+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:49:21.069] clusterConfig unchanged [2021-10-22T00:49:21.184+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:49:21.184] ...Finished start [pid=50100] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38508,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:49:21.306+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:49:21.306] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 121.820676ms. [2021-10-22T00:49:21.306+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:49:21.306] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38508 (local=false) [2021-10-22T00:49:21.306+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:49:21.306] Dropping existing documents from system.replset [2021-10-22T00:49:21.426+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:49:21.426] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:49:21.429+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:49:21.429] Inserting oplogTruncateAfterPoint document [2021-10-22T00:49:21.429+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:49:21.429] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:49:21.430+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:49:21.430] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:49:21.430+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:49:21.430] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:49:21.443+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:49:21.443] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:49:21.443+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:49:21.443] Setting the minvalid document to [{_id ObjectID("61720a91523370002becc37b")} {t -1} {ts {0 1}}] [2021-10-22T00:49:21.456+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:49:21.456] Inserted [{_id ObjectID("61720a91523370002becc37b")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:49:21.457+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:49:21.457] Inserting new replica set configuration document. [2021-10-22T00:49:21.457+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:49:21.457] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:49:21.469+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:49:21.468] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:49:21.472+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:49:21.472] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:49:21.472+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:49:21.472] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:49:21.472+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:49:21.472] Initiating stopEphemeralMongo [2021-10-22T00:49:21.472+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:49:21.472] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38508 (local=false) [2021-10-22T00:49:21.472+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:49:21.472] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38508 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:49:21.604+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:21.604] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:21.913+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:49:21.913] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.030881ms. [2021-10-22T00:49:21.913+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:49:21.913] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38508 (local=false) [2021-10-22T00:49:21.913+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:49:21.913] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.354µs. [2021-10-22T00:49:21.913+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:49:21.913] Successfully finished running stopEphemeralMongo [2021-10-22T00:49:21.913+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:49:21.913] Successfully stopped existing ephemeral mongod. [2021-10-22T00:49:21.914+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:49:21.913] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:49:21.914+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:49:21.914] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:49:21.948+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:49:21.948] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:49:21.949+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:49:21.949] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48252 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:49:22.173+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:49:22.173] clusterConfig unchanged [2021-10-22T00:49:22.604+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:22.604] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:23.159+0000] [.error] [src/action/start.go:func1:93] [103] [00:49:23.159] Error starting mongod : [00:49:23.159] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48252,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48252 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 51613 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:49:21.999+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:49:21.999+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:49:21.999+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48252,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:49:22.000+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:49:22.002+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:49:22.003+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:49:22.042+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:49:22.042+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:49:22.053+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:49:22.101+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:49:22.101+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:49:22.644+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863762:644181][51613:0x7fbb61ef0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:49:22.758+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863762:758773][51613:0x7fbb61ef0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:49:22.858+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863762:858594][51613:0x7fbb61ef0bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:49:22.972+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863762:972105][51613:0x7fbb61ef0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:49:23.047+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863763:47699][51613:0x7fbb61ef0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:49:23.106+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863763:106843][51613:0x7fbb61ef0bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:49:23.106+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863763:106921][51613:0x7fbb61ef0bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:49:23.108+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:49:23.159+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:49:23.159] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48252 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:49:23.159] Error starting mongod : [00:49:23.159] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48252,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48252 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 51613 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:49:21.999+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:49:21.999+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:49:21.999+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48252,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:49:22.000+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:49:22.002+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:49:22.003+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:49:22.042+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:49:22.042+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:49:22.053+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:49:22.101+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:49:22.101+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:49:22.644+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863762:644181][51613:0x7fbb61ef0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:49:22.758+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863762:758773][51613:0x7fbb61ef0bc0], txn-recover: [W...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:49:23.159+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:49:23.159] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:49:23.159] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48252 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:49:23.159] Error starting mongod : [00:49:23.159] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48252,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48252 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 51613 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:49:21.999+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:49:21.999+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:49:21.999+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48252,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:49:22.000+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:49:22.002+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:49:22.003+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:49:22.042+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:49:22.042+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:49:22.053+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:49:22.101+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:49:22.101+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:49:22.644+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863762:644181][51613:0x7fbb61ef0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:49:22.758+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ct... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:49:23.160+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:49:23.160] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:49:23.160+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:49:23.160] Initiating stopEphemeralMongo [2021-10-22T00:49:23.160+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:49:23.160] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:49:23.160+0000] [.error] [src/director/director.go:executePlan:962] [00:49:23.160] Failed to apply action. Result = : [00:49:23.159] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:49:23.159] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48252 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:49:23.159] Error starting mongod : [00:49:23.159] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48252,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48252 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 51613 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:49:21.999+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:49:21.999+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:49:21.999+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48252,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:49:22.000+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:49:22.002+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:49:22.003+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:49:22.042+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:49:22.042+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:49:22.053+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:49:22.101+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:49:22.101+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:49:22.644+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863762:644181][51613:0x7fbb61ef0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:49:23.160+0000] [.error] [src/director/director.go:planAndExecute:567] [00:49:23.160] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:49:23.160] Failed to apply action. Result = : [00:49:23.159] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:49:23.159] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48252 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:49:23.159] Error starting mongod : [00:49:23.159] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48252,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48252 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 51613 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:49:21.999+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:49:21.999+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:49:21.999+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48252,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:49:22.000+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:49:22.002+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:49:22.003+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:49:22.042+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:49:22.042+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:49:22.053+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:49:22.101+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:49:22.101+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:49:22.644+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:49:23.160+0000] [.error] [src/director/director.go:mainLoop:399] [00:49:23.160] Failed to planAndExecute : [00:49:23.160] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:49:23.160] Failed to apply action. Result = : [00:49:23.159] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:49:23.159] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48252 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:49:23.159] Error starting mongod : [00:49:23.159] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48252,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48252 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 51613 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:49:21.999+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:49:21.999+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:49:21.999+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48252,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:49:22.000+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:49:22.002+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:49:22.003+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:49:22.042+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:49:22.042+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:49:22.053+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:49:22.101+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:49:22.101+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:49:22.644+00:00"},"s":"I", "c":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:49:23.482+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:49:23.481] clusterConfig unchanged [2021-10-22T00:49:23.605+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:23.605] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:24.605+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:24.605] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:25.606+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:25.606] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:26.604+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:26.604] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:27.606+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:27.606] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:28.607+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:28.607] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:29.609+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:29.608] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:30.607+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:30.607] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:31.608+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:31.608] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:31.885+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:49:31.885] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.286995021s [2021-10-22T00:49:32.607+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:32.607] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:33.608+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:33.608] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:33.613+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:49:33.613] clusterConfig unchanged [2021-10-22T00:49:34.609+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:34.609] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:35.610+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:35.610] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:36.610+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:36.610] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:37.611+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:37.611] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:38.612+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:38.612] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:39.613+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:39.613] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:40.613+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:40.613] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:41.266+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:49:41.266] memory:rss=51404800 alloc=13119760 totalalloc=2581344040 mallocs=33464304 frees=33357565 heapinuse=16490496 heapobjects=106739 gcpausetotalns=1407176888 gcpauselastns=102744 gcnum=413 utimens=44641181000 stimens=13361654000 minflt=122893 majflt=405 [2021-10-22T00:49:41.618+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:41.618] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:42.614+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:42.614] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:43.614+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:43.614] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:43.805+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:49:43.805] clusterConfig unchanged [2021-10-22T00:49:44.615+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:44.615] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:45.616+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:45.616] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:46.617+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:46.617] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:47.618+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:47.618] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:48.618+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:48.618] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:49.618+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:49.618] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:50.623+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:50.623] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:51.619+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:51.619] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:52.620+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:52.620] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:53.621+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:53.621] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:53.927+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:49:53.927] clusterConfig unchanged [2021-10-22T00:49:54.621+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:54.621] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:55.622+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:55.622] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:56.622+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:56.622] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:57.623+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:57.623] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:58.624+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:58.623] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:49:59.624+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:49:59.624] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:00.625+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:00.625] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:01.632+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:01.632] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:02.628+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:02.628] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:03.631+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:03.631] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:04.019+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:50:04.019] clusterConfig unchanged [2021-10-22T00:50:04.629+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:04.629] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:05.629+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:05.629] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:06.630+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:06.630] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:07.631+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:07.631] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:08.634+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:08.634] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:09.632+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:09.632] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:10.632+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:10.632] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:11.633+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:11.633] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:12.637+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:12.637] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:13.634+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:13.634] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:14.121+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:50:14.121] clusterConfig unchanged [2021-10-22T00:50:14.635+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:14.635] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:15.635+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:15.635] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:16.635+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:16.635] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:17.636+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:17.636] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:18.637+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:18.637] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:19.637+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:19.637] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:20.638+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:20.638] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:21.638+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:21.638] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:22.639+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:22.639] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:23.333+0000] [.info] [src/director/director.go:computePlan:280] [00:50:23.333] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:50:23.333+0000] [.info] [src/director/director.go:tracef:794] [00:50:23.333] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:50:23.333+0000] [.info] [src/director/director.go:tracef:794] [00:50:23.333] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:50:23.333+0000] [.info] [src/director/director.go:tracef:794] [00:50:23.333] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:50:23.333+0000] [.info] [src/director/director.go:tracef:794] [00:50:23.333] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:50:23.334+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:50:23.334] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:50:23.506+0000] [.info] [src/director/director.go:tracef:794] [00:50:23.506] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:50:23.506+0000] [.info] [src/director/director.go:tracef:794] [00:50:23.506] because [2021-10-22T00:50:23.506+0000] [.info] [src/director/director.go:tracef:794] [00:50:23.506] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:50:23.506+0000] [.info] [src/director/director.go:tracef:794] [00:50:23.506] because ['currentState.CanBounceStop' = true] [2021-10-22T00:50:23.642+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:23.642] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:23.666+0000] [.info] [src/director/director.go:tracef:794] [00:50:23.666] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:50:23.666+0000] [.info] [src/director/director.go:tracef:794] [00:50:23.666] because [2021-10-22T00:50:23.667+0000] [.info] [src/director/director.go:tracef:794] [00:50:23.667] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:50:23.667+0000] [.info] [src/director/director.go:tracef:794] [00:50:23.667] because ['currentState.Up' = false] [2021-10-22T00:50:23.667+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:50:23.667] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:50:23.667+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:50:23.667] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:50:23.751+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:50:23.751] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:50:23.759+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:50:23.759] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:50:23.760+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:50:23.760] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:50:23.762+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:50:23.762] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:50:23.762+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:50:23.762] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:50:23.762+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:50:23.762] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:50:23.763+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:50:23.763] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:50:23.988+0000] [.info] [src/director/director.go:tracef:794] [00:50:23.987] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:50:23.988+0000] [.info] [src/director/director.go:tracef:794] [00:50:23.988] because ['currentState.Up' = false] [2021-10-22T00:50:24.213+0000] [.info] [src/director/director.go:tracef:794] [00:50:24.213] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:50:24.213+0000] [.info] [src/director/director.go:tracef:794] [00:50:24.213] because ['currentState.Up' = false] [2021-10-22T00:50:24.213+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:50:24.213] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:50:24.312+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:50:24.312] clusterConfig unchanged [2021-10-22T00:50:24.440+0000] [.info] [src/director/director.go:tracef:794] [00:50:24.440] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:50:24.440+0000] [.info] [src/director/director.go:tracef:794] [00:50:24.440] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:50:24.643+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:24.643] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:24.654+0000] [.info] [src/director/director.go:tracef:794] [00:50:24.654] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:50:24.654+0000] [.info] [src/director/director.go:tracef:794] [00:50:24.654] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:50:24.654+0000] [.info] [src/director/director.go:tracef:794] [00:50:24.654] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:50:24.654+0000] [.info] [src/director/director.go:tracef:794] [00:50:24.654] because ['currentState.Up' = false] [2021-10-22T00:50:24.654+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:50:24.654] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:50:24.654+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:50:24.654] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:50:24.692+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:50:24.692] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:50:24.693+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:50:24.693] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53199 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:50:25.409+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:50:25.409] ...Finished start [pid=70975] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53199,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:50:25.434+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:50:25.434] clusterConfig unchanged [2021-10-22T00:50:25.535+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:50:25.535] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 125.348947ms. [2021-10-22T00:50:25.535+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:50:25.535] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53199 (local=false) [2021-10-22T00:50:25.535+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:50:25.535] Dropping existing documents from system.replset [2021-10-22T00:50:25.643+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:25.643] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:25.662+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:50:25.662] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:50:25.666+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:50:25.666] Inserting oplogTruncateAfterPoint document [2021-10-22T00:50:25.666+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:50:25.666] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:50:25.667+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:50:25.667] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:50:25.667+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:50:25.667] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:50:25.680+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:50:25.680] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:50:25.680+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:50:25.680] Setting the minvalid document to [{_id ObjectID("61720ad1523370002becc389")} {t -1} {ts {0 1}}] [2021-10-22T00:50:25.693+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:50:25.693] Inserted [{_id ObjectID("61720ad1523370002becc389")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:50:25.693+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:50:25.693] Inserting new replica set configuration document. [2021-10-22T00:50:25.693+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:50:25.693] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:50:25.707+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:50:25.707] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:50:25.711+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:50:25.711] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:50:25.711+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:50:25.711] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:50:25.711+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:50:25.711] Initiating stopEphemeralMongo [2021-10-22T00:50:25.711+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:50:25.711] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53199 (local=false) [2021-10-22T00:50:25.711+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:50:25.711] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53199 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:50:26.062+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:50:26.062] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 4.738176ms. [2021-10-22T00:50:26.062+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:50:26.062] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53199 (local=false) [2021-10-22T00:50:26.062+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:50:26.062] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.092µs. [2021-10-22T00:50:26.062+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:50:26.062] Successfully finished running stopEphemeralMongo [2021-10-22T00:50:26.063+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:50:26.062] Successfully stopped existing ephemeral mongod. [2021-10-22T00:50:26.063+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:50:26.063] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:50:26.063+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:50:26.063] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:50:26.095+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:50:26.095] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:50:26.096+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:50:26.096] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59996 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:50:26.557+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:50:26.557] clusterConfig unchanged [2021-10-22T00:50:26.644+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:26.644] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:27.278+0000] [.error] [src/action/start.go:func1:93] [103] [00:50:27.278] Error starting mongod : [00:50:27.277] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59996,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59996 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:50:26.127Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 72457 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:50:26.146+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:50:26.146+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:50:26.146+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59996,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:50:26.148+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:50:26.150+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:50:26.150+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:50:26.220+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:50:26.220+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:50:26.221+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:50:26.267+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:50:26.267+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:50:26.762+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863826:762831][72457:0x7f5cf2065bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:50:26.864+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863826:864137][72457:0x7f5cf2065bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:50:26.964+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863826:964814][72457:0x7f5cf2065bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T00:50:27.077+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863827:77088][72457:0x7f5cf2065bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:50:27.151+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863827:151842][72457:0x7f5cf2065bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:50:27.214+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863827:214509][72457:0x7f5cf2065bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:50:27.214+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863827:214581][72457:0x7f5cf2065bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:50:27.278+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:50:27.278] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59996 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:50:27.278] Error starting mongod : [00:50:27.277] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59996,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59996 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:50:26.127Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 72457 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:50:26.146+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:50:26.146+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:50:26.146+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59996,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:50:26.148+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:50:26.150+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:50:26.150+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:50:26.220+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:50:26.220+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:50:26.221+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:50:26.267+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:50:26.267+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:50:26.762+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863826:762831][72457:0x7f5cf2065bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:50:26.864+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:50:27.278+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:50:27.278] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:50:27.278] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59996 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:50:27.278] Error starting mongod : [00:50:27.277] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59996,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59996 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:50:26.127Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 72457 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:50:26.146+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:50:26.146+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:50:26.146+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59996,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:50:26.148+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:50:26.150+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:50:26.150+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:50:26.220+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:50:26.220+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:50:26.221+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:50:26.267+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:50:26.267+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:50:26.762+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863826:762831][72457:0x7f5cf2065bc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:50:27.278+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:50:27.278] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:50:27.278+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:50:27.278] Initiating stopEphemeralMongo [2021-10-22T00:50:27.278+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:50:27.278] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:50:27.279+0000] [.error] [src/director/director.go:executePlan:962] [00:50:27.279] Failed to apply action. Result = : [00:50:27.278] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:50:27.278] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59996 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:50:27.278] Error starting mongod : [00:50:27.277] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59996,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59996 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:50:26.127Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 72457 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:50:26.146+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:50:26.146+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:50:26.146+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59996,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:50:26.148+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:50:26.150+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:50:26.150+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:50:26.220+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:50:26.220+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:50:26.221+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:50:26.267+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:50:26.267+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:50:26.762+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:50:27.279+0000] [.error] [src/director/director.go:planAndExecute:567] [00:50:27.279] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:50:27.279] Failed to apply action. Result = : [00:50:27.278] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:50:27.278] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59996 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:50:27.278] Error starting mongod : [00:50:27.277] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59996,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59996 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:50:26.127Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 72457 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:50:26.146+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:50:26.146+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:50:26.146+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59996,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:50:26.148+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:50:26.150+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:50:26.150+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:50:26.220+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:50:26.220+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:50:26.221+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:50:26.267+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:50:26.267+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:50:27.279+0000] [.error] [src/director/director.go:mainLoop:399] [00:50:27.279] Failed to planAndExecute : [00:50:27.279] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:50:27.279] Failed to apply action. Result = : [00:50:27.278] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:50:27.278] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59996 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:50:27.278] Error starting mongod : [00:50:27.277] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59996,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59996 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:50:26.127Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 72457 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:50:26.146+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:50:26.146+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:50:26.146+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59996,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:50:26.148+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:50:26.150+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:50:26.150+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:50:26.220+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:50:26.220+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:50:26.221+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:50:26.267+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:50:26.267+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:50:27.645+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:27.645] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:27.808+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:50:27.808] clusterConfig unchanged [2021-10-22T00:50:28.645+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:28.645] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:29.647+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:29.647] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:30.648+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:30.648] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:31.646+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:31.646] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:32.648+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:32.648] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:33.649+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:33.649] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:34.648+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:34.648] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:35.648+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:35.648] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:35.914+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:50:35.914] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.151163175s [2021-10-22T00:50:36.648+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:36.648] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:37.653+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:37.653] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:37.940+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:50:37.940] clusterConfig unchanged [2021-10-22T00:50:38.648+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:38.648] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:39.648+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:39.648] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:40.649+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:40.649] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:41.267+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:50:41.267] memory:rss=50647040 alloc=8545520 totalalloc=2666732816 mallocs=34542698 frees=34482293 heapinuse=14180352 heapobjects=60405 gcpausetotalns=1409303588 gcpauselastns=106219 gcnum=427 utimens=46171429000 stimens=13869581000 minflt=126774 majflt=405 [2021-10-22T00:50:41.650+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:41.650] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:42.650+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:42.650] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:43.651+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:43.651] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:44.156+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [00:50:44.156] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:59996 (local=false) because it is not up [2021-10-22T00:50:44.652+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:44.652] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:45.652+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:45.652] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:46.652+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:46.652] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:47.653+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:47.653] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:48.030+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:50:48.030] clusterConfig unchanged [2021-10-22T00:50:48.653+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:48.653] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:49.654+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:49.654] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:50.654+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:50.654] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:51.655+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:51.655] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:52.655+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:52.655] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:53.656+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:53.656] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:54.657+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:54.656] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:55.658+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:55.658] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:56.658+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:56.658] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:57.659+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:57.659] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:58.172+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:50:58.172] clusterConfig unchanged [2021-10-22T00:50:58.659+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:58.659] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:50:59.660+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:50:59.660] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:00.660+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:00.660] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:01.661+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:01.661] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:02.662+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:02.662] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:03.662+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:03.661] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:04.662+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:04.662] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:05.669+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:05.669] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:06.664+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:06.664] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:07.664+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:07.664] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:08.274+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:51:08.274] clusterConfig unchanged [2021-10-22T00:51:08.664+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:08.664] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:09.665+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:09.665] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:10.666+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:10.665] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:11.666+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:11.666] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:12.667+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:12.667] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:13.667+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:13.667] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:14.668+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:14.668] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:15.669+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:15.669] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:16.675+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:16.675] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:17.672+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:17.672] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:18.386+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:51:18.386] clusterConfig unchanged [2021-10-22T00:51:18.556+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [00:51:18.556] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T00:51:18.558+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:18.558] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T00:51:18.558+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [00:51:18.558] Could not get version : [00:51:18.558] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T00:51:18.558+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [00:51:18.558] Error getting version for log lines. Skipping : [00:51:18.558] Could not get version : [00:51:18.558] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T00:51:18.671+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:18.671] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:19.672+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:19.672] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:20.672+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:20.672] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:21.673+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:21.673] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:22.674+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:22.674] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:23.674+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:23.674] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:24.675+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:24.675] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:25.682+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:25.682] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:26.676+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:26.676] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:27.445+0000] [.info] [src/director/director.go:computePlan:280] [00:51:27.445] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:51:27.445+0000] [.info] [src/director/director.go:tracef:794] [00:51:27.445] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:51:27.445+0000] [.info] [src/director/director.go:tracef:794] [00:51:27.445] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:51:27.445+0000] [.info] [src/director/director.go:tracef:794] [00:51:27.445] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:51:27.445+0000] [.info] [src/director/director.go:tracef:794] [00:51:27.445] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:51:27.445+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:51:27.445] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:51:27.610+0000] [.info] [src/director/director.go:tracef:794] [00:51:27.610] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:51:27.610+0000] [.info] [src/director/director.go:tracef:794] [00:51:27.610] because [2021-10-22T00:51:27.610+0000] [.info] [src/director/director.go:tracef:794] [00:51:27.610] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:51:27.610+0000] [.info] [src/director/director.go:tracef:794] [00:51:27.610] because ['currentState.CanBounceStop' = true] [2021-10-22T00:51:27.680+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:27.680] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:27.763+0000] [.info] [src/director/director.go:tracef:794] [00:51:27.763] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:51:27.763+0000] [.info] [src/director/director.go:tracef:794] [00:51:27.763] because [2021-10-22T00:51:27.763+0000] [.info] [src/director/director.go:tracef:794] [00:51:27.763] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:51:27.763+0000] [.info] [src/director/director.go:tracef:794] [00:51:27.763] because ['currentState.Up' = false] [2021-10-22T00:51:27.763+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:51:27.763] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:51:27.763+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:51:27.763] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:51:27.849+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:51:27.849] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:51:27.860+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:51:27.860] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:51:27.861+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:51:27.861] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:51:27.862+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:51:27.862] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:51:27.863+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:51:27.863] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:51:27.863+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:51:27.863] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:51:27.863+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:51:27.863] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:51:27.865+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:51:27.865] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:51:28.092+0000] [.info] [src/director/director.go:tracef:794] [00:51:28.092] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:51:28.092+0000] [.info] [src/director/director.go:tracef:794] [00:51:28.092] because ['currentState.Up' = false] [2021-10-22T00:51:28.281+0000] [.info] [src/director/director.go:tracef:794] [00:51:28.281] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:51:28.281+0000] [.info] [src/director/director.go:tracef:794] [00:51:28.281] because ['currentState.Up' = false] [2021-10-22T00:51:28.281+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:51:28.281] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:51:28.493+0000] [.info] [src/director/director.go:tracef:794] [00:51:28.493] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:51:28.493+0000] [.info] [src/director/director.go:tracef:794] [00:51:28.493] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:51:28.563+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:51:28.563] clusterConfig unchanged [2021-10-22T00:51:28.694+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:28.694] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:28.712+0000] [.info] [src/director/director.go:tracef:794] [00:51:28.712] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:51:28.712+0000] [.info] [src/director/director.go:tracef:794] [00:51:28.712] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:51:28.712+0000] [.info] [src/director/director.go:tracef:794] [00:51:28.712] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:51:28.712+0000] [.info] [src/director/director.go:tracef:794] [00:51:28.712] because ['currentState.Up' = false] [2021-10-22T00:51:28.712+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:51:28.712] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:51:28.712+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:51:28.712] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:51:28.749+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:51:28.749] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:51:28.749+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:51:28.749] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48139 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:51:29.462+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:51:29.462] ...Finished start [pid=17570] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48139,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:51:29.580+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:51:29.580] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 118.146558ms. [2021-10-22T00:51:29.580+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:51:29.580] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:48139 (local=false) [2021-10-22T00:51:29.580+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:51:29.580] Dropping existing documents from system.replset [2021-10-22T00:51:29.680+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:51:29.680] clusterConfig unchanged [2021-10-22T00:51:29.705+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:29.705] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:29.731+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:51:29.731] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:51:29.737+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:51:29.737] Inserting oplogTruncateAfterPoint document [2021-10-22T00:51:29.737+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:51:29.737] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:51:29.738+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:51:29.738] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:51:29.738+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:51:29.738] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:51:29.751+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:51:29.751] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:51:29.751+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:51:29.751] Setting the minvalid document to [{_id ObjectID("61720b11523370002becc397")} {t -1} {ts {0 1}}] [2021-10-22T00:51:29.763+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:51:29.763] Inserted [{_id ObjectID("61720b11523370002becc397")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:51:29.763+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:51:29.763] Inserting new replica set configuration document. [2021-10-22T00:51:29.763+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:51:29.763] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:51:29.777+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:51:29.777] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:51:29.780+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:51:29.780] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:51:29.780+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:51:29.780] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:51:29.780+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:51:29.780] Initiating stopEphemeralMongo [2021-10-22T00:51:29.781+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:51:29.780] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:48139 (local=false) [2021-10-22T00:51:29.781+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:51:29.781] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:48139 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:51:30.126+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:51:30.126] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.254721ms. [2021-10-22T00:51:30.126+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:51:30.126] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:48139 (local=false) [2021-10-22T00:51:30.126+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:51:30.126] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 39.57µs. [2021-10-22T00:51:30.126+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:51:30.126] Successfully finished running stopEphemeralMongo [2021-10-22T00:51:30.126+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:51:30.126] Successfully stopped existing ephemeral mongod. [2021-10-22T00:51:30.126+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:51:30.126] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:51:30.126+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:51:30.126] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:51:30.162+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:51:30.162] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:51:30.162+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:51:30.162] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47175 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:51:30.692+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:30.692] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:30.792+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:51:30.792] clusterConfig unchanged [2021-10-22T00:51:31.404+0000] [.error] [src/action/start.go:func1:93] [103] [00:51:31.404] Error starting mongod : [00:51:31.403] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47175,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47175 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 19161 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:51:30.225+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:51:30.225+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:51:30.225+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47175,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:51:30.227+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:51:30.229+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:51:30.230+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:51:30.284+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:51:30.284+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:51:30.285+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:51:30.332+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:51:30.332+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:51:30.866+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863890:866920][19161:0x7fa69f637bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:51:30.983+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863890:983113][19161:0x7fa69f637bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:51:31.080+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863891:80761][19161:0x7fa69f637bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:51:31.193+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863891:193756][19161:0x7fa69f637bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:51:31.290+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863891:290278][19161:0x7fa69f637bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:51:31.350+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863891:350025][19161:0x7fa69f637bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:51:31.350+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863891:350097][19161:0x7fa69f637bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:51:31.351+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:51:31.404+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:51:31.404] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:47175 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:51:31.404] Error starting mongod : [00:51:31.403] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47175,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47175 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 19161 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:51:30.225+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:51:30.225+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:51:30.225+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47175,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:51:30.227+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:51:30.229+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:51:30.230+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:51:30.284+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:51:30.284+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:51:30.285+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:51:30.332+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:51:30.332+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:51:30.866+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863890:866920][19161:0x7fa69f637bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:51:30.983+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863890:983113][19161:0x7fa69f637bc0], txn-recover: [W...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:51:31.404+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:51:31.404] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:51:31.404] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:47175 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:51:31.404] Error starting mongod : [00:51:31.403] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47175,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47175 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 19161 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:51:30.225+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:51:30.225+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:51:30.225+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47175,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:51:30.227+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:51:30.229+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:51:30.230+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:51:30.284+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:51:30.284+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:51:30.285+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:51:30.332+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:51:30.332+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:51:30.866+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863890:866920][19161:0x7fa69f637bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:51:30.983+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ct... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:51:31.404+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:51:31.404] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:51:31.404+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:51:31.404] Initiating stopEphemeralMongo [2021-10-22T00:51:31.404+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:51:31.404] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:51:31.407+0000] [.error] [src/director/director.go:executePlan:962] [00:51:31.407] Failed to apply action. Result = : [00:51:31.404] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:51:31.404] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:47175 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:51:31.404] Error starting mongod : [00:51:31.403] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47175,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47175 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 19161 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:51:30.225+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:51:30.225+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:51:30.225+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47175,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:51:30.227+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:51:30.229+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:51:30.230+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:51:30.284+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:51:30.284+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:51:30.285+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:51:30.332+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:51:30.332+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:51:30.866+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863890:866920][19161:0x7fa69f637bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:51:31.408+0000] [.error] [src/director/director.go:planAndExecute:567] [00:51:31.407] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:51:31.407] Failed to apply action. Result = : [00:51:31.404] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:51:31.404] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:47175 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:51:31.404] Error starting mongod : [00:51:31.403] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47175,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47175 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 19161 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:51:30.225+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:51:30.225+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:51:30.225+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47175,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:51:30.227+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:51:30.229+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:51:30.230+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:51:30.284+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:51:30.284+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:51:30.285+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:51:30.332+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:51:30.332+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:51:30.866+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:51:31.408+0000] [.error] [src/director/director.go:mainLoop:399] [00:51:31.408] Failed to planAndExecute : [00:51:31.407] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:51:31.407] Failed to apply action. Result = : [00:51:31.404] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:51:31.404] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:47175 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:51:31.404] Error starting mongod : [00:51:31.403] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47175,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47175 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 19161 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:51:30.225+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:51:30.225+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:51:30.225+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47175,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:51:30.227+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:51:30.229+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:51:30.230+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:51:30.284+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:51:30.284+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:51:30.285+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:51:30.332+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:51:30.332+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:51:30.866+00:00"},"s":"I", "c":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:51:31.690+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:31.690] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:32.054+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:51:32.054] clusterConfig unchanged [2021-10-22T00:51:32.690+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:32.690] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:33.693+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:33.693] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:34.693+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:34.693] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:35.692+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:35.692] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:36.694+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:36.694] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:37.693+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:37.693] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:38.695+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:38.695] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:39.697+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:39.697] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:40.149+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:51:40.148] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.283224174s [2021-10-22T00:51:40.694+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:40.694] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:41.268+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:51:41.268] memory:rss=51134464 alloc=9192800 totalalloc=2753714240 mallocs=35623553 frees=35558139 heapinuse=14614528 heapobjects=65414 gcpausetotalns=1411220423 gcpauselastns=109570 gcnum=441 utimens=47709591000 stimens=14429733000 minflt=131290 majflt=405 [2021-10-22T00:51:41.695+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:41.695] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:42.283+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:51:42.283] clusterConfig unchanged [2021-10-22T00:51:42.696+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:42.696] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:43.697+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:43.697] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:44.697+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:44.697] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:45.698+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:45.698] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:46.698+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:46.698] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:47.703+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:47.703] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:48.699+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:48.699] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:49.700+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:49.700] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:50.701+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:50.701] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:51.701+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:51.701] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:52.415+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:51:52.415] clusterConfig unchanged [2021-10-22T00:51:52.704+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:52.704] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:53.703+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:53.703] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:54.704+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:54.704] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:55.706+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:55.706] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:56.709+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:56.709] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:57.706+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:57.706] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:58.707+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:58.707] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:51:59.707+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:51:59.707] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:00.708+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:00.708] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:01.709+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:01.709] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:02.517+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:52:02.517] clusterConfig unchanged [2021-10-22T00:52:02.709+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:02.709] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:03.710+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:03.709] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:04.710+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:04.710] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:05.711+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:05.711] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:06.712+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:06.712] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:07.713+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:07.713] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:08.718+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:08.718] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:09.713+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:09.713] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:10.717+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:10.717] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:11.715+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:11.715] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:12.708+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:52:12.708] clusterConfig unchanged [2021-10-22T00:52:12.716+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:12.716] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:13.717+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:13.717] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:14.717+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:14.717] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:15.717+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:15.717] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:16.719+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:16.719] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:17.718+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:17.718] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:18.719+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:18.719] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:19.724+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:19.724] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:20.722+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:20.722] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:21.720+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:21.720] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:22.720+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:22.720] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:22.830+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:52:22.830] clusterConfig unchanged [2021-10-22T00:52:23.720+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:23.720] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:24.721+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:24.721] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:25.725+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:25.725] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:26.723+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:26.723] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:27.557+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [00:52:27.557] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T00:52:27.560+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [00:52:27.560] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T00:52:27.723+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:27.723] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:28.723+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:28.723] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:29.724+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:29.724] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:30.725+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:30.725] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:31.578+0000] [.info] [src/director/director.go:computePlan:280] [00:52:31.578] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:52:31.578+0000] [.info] [src/director/director.go:tracef:794] [00:52:31.578] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:52:31.578+0000] [.info] [src/director/director.go:tracef:794] [00:52:31.578] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:52:31.578+0000] [.info] [src/director/director.go:tracef:794] [00:52:31.578] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:52:31.578+0000] [.info] [src/director/director.go:tracef:794] [00:52:31.578] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:52:31.579+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:52:31.579] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:52:31.730+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:31.730] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:31.732+0000] [.info] [src/director/director.go:tracef:794] [00:52:31.732] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:52:31.732+0000] [.info] [src/director/director.go:tracef:794] [00:52:31.732] because [2021-10-22T00:52:31.732+0000] [.info] [src/director/director.go:tracef:794] [00:52:31.732] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:52:31.733+0000] [.info] [src/director/director.go:tracef:794] [00:52:31.732] because ['currentState.CanBounceStop' = true] [2021-10-22T00:52:31.884+0000] [.info] [src/director/director.go:tracef:794] [00:52:31.884] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:52:31.884+0000] [.info] [src/director/director.go:tracef:794] [00:52:31.884] because [2021-10-22T00:52:31.884+0000] [.info] [src/director/director.go:tracef:794] [00:52:31.884] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:52:31.884+0000] [.info] [src/director/director.go:tracef:794] [00:52:31.884] because ['currentState.Up' = false] [2021-10-22T00:52:31.884+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:52:31.884] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:52:31.884+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:52:31.884] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:52:31.967+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:52:31.967] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:52:31.967+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:52:31.967] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:52:31.975+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:52:31.975] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:52:31.977+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:52:31.977] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:52:31.978+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:52:31.978] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:52:31.978+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:52:31.978] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:52:31.978+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:52:31.978] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:52:31.980+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:52:31.980] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:52:32.197+0000] [.info] [src/director/director.go:tracef:794] [00:52:32.197] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:52:32.197+0000] [.info] [src/director/director.go:tracef:794] [00:52:32.197] because ['currentState.Up' = false] [2021-10-22T00:52:32.389+0000] [.info] [src/director/director.go:tracef:794] [00:52:32.389] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:52:32.389+0000] [.info] [src/director/director.go:tracef:794] [00:52:32.389] because ['currentState.Up' = false] [2021-10-22T00:52:32.389+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:52:32.389] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:52:32.609+0000] [.info] [src/director/director.go:tracef:794] [00:52:32.609] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:52:32.610+0000] [.info] [src/director/director.go:tracef:794] [00:52:32.609] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:52:32.736+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:32.736] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:32.834+0000] [.info] [src/director/director.go:tracef:794] [00:52:32.834] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:52:32.834+0000] [.info] [src/director/director.go:tracef:794] [00:52:32.834] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:52:32.835+0000] [.info] [src/director/director.go:tracef:794] [00:52:32.835] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:52:32.835+0000] [.info] [src/director/director.go:tracef:794] [00:52:32.835] because ['currentState.Up' = false] [2021-10-22T00:52:32.835+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:52:32.835] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:52:32.835+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:52:32.835] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:52:32.870+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:52:32.870] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:52:32.870+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:52:32.870] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40966 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:52:33.060+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:52:33.060] clusterConfig unchanged [2021-10-22T00:52:33.587+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:52:33.587] ...Finished start [pid=38454] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40966,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:52:33.706+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:52:33.706] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 119.144792ms. [2021-10-22T00:52:33.706+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:52:33.706] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40966 (local=false) [2021-10-22T00:52:33.706+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:52:33.706] Dropping existing documents from system.replset [2021-10-22T00:52:33.734+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:33.734] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:33.839+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:52:33.839] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:52:33.847+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:52:33.847] Inserting oplogTruncateAfterPoint document [2021-10-22T00:52:33.847+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:52:33.847] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:52:33.848+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:52:33.848] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:52:33.848+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:52:33.848] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:52:33.860+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:52:33.860] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:52:33.860+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:52:33.860] Setting the minvalid document to [{_id ObjectID("61720b51523370002becc3a5")} {t -1} {ts {0 1}}] [2021-10-22T00:52:33.873+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:52:33.873] Inserted [{_id ObjectID("61720b51523370002becc3a5")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:52:33.873+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:52:33.873] Inserting new replica set configuration document. [2021-10-22T00:52:33.873+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:52:33.873] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:52:33.887+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:52:33.887] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:52:33.891+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:52:33.891] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:52:33.891+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:52:33.891] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:52:33.891+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:52:33.891] Initiating stopEphemeralMongo [2021-10-22T00:52:33.891+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:52:33.891] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40966 (local=false) [2021-10-22T00:52:33.891+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:52:33.891] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40966 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:52:34.182+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:52:34.182] clusterConfig unchanged [2021-10-22T00:52:34.394+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:52:34.394] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.074174ms. [2021-10-22T00:52:34.394+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:52:34.394] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40966 (local=false) [2021-10-22T00:52:34.394+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:52:34.394] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.081µs. [2021-10-22T00:52:34.394+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:52:34.394] Successfully finished running stopEphemeralMongo [2021-10-22T00:52:34.394+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:52:34.394] Successfully stopped existing ephemeral mongod. [2021-10-22T00:52:34.394+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:52:34.394] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:52:34.394+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:52:34.394] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:52:34.428+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:52:34.428] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:52:34.429+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:52:34.429] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36660 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:52:34.739+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [00:52:34.739] lsblk cache invalidated, running lsblk command [2021-10-22T00:52:34.742+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:34.742] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:35.320+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:52:35.320] clusterConfig unchanged [2021-10-22T00:52:35.667+0000] [.error] [src/action/start.go:func1:93] [103] [00:52:35.667] Error starting mongod : [00:52:35.666] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36660,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36660 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 40092 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:52:34.481+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:52:34.481+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:52:34.481+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36660,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:52:34.483+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:52:34.484+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:52:34.485+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:52:34.504+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:52:34.504+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:52:34.528+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:52:34.575+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:52:34.575+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:52:35.076+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863955:76142][40092:0x7f9f05eaebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:52:35.199+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863955:199381][40092:0x7f9f05eaebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:52:35.356+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863955:356717][40092:0x7f9f05eaebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/33920 to 2/256"}} {"t":{"$date":"2021-10-22T00:52:35.479+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863955:479111][40092:0x7f9f05eaebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:52:35.549+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863955:549012][40092:0x7f9f05eaebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:52:35.614+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863955:614072][40092:0x7f9f05eaebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:52:35.614+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863955:614127][40092:0x7f9f05eaebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:52:35.615+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:52:35.667+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:52:35.667] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36660 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:52:35.667] Error starting mongod : [00:52:35.666] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36660,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36660 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 40092 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:52:34.481+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:52:34.481+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:52:34.481+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36660,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:52:34.483+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:52:34.484+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:52:34.485+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:52:34.504+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:52:34.504+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:52:34.528+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:52:34.575+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:52:34.575+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:52:35.076+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863955:76142][40092:0x7f9f05eaebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:52:35.199+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863955:199381][40092:0x7f9f05eaebc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:52:35.668+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:52:35.668] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:52:35.667] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36660 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:52:35.667] Error starting mongod : [00:52:35.666] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36660,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36660 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 40092 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:52:34.481+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:52:34.481+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:52:34.481+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36660,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:52:34.483+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:52:34.484+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:52:34.485+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:52:34.504+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:52:34.504+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:52:34.528+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:52:34.575+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:52:34.575+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:52:35.076+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863955:76142][40092:0x7f9f05eaebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:52:35.199+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:52:35.668+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:52:35.668] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:52:35.668+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:52:35.668] Initiating stopEphemeralMongo [2021-10-22T00:52:35.668+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:52:35.668] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:52:35.670+0000] [.error] [src/director/director.go:executePlan:962] [00:52:35.670] Failed to apply action. Result = : [00:52:35.668] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:52:35.667] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36660 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:52:35.667] Error starting mongod : [00:52:35.666] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36660,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36660 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 40092 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:52:34.481+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:52:34.481+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:52:34.481+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36660,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:52:34.483+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:52:34.484+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:52:34.485+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:52:34.504+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:52:34.504+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:52:34.528+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:52:34.575+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:52:34.575+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:52:35.076+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634863955:76142][40092:0x7f9f05eaebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:52:35.670+0000] [.error] [src/director/director.go:planAndExecute:567] [00:52:35.670] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:52:35.670] Failed to apply action. Result = : [00:52:35.668] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:52:35.667] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36660 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:52:35.667] Error starting mongod : [00:52:35.666] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36660,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36660 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 40092 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:52:34.481+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:52:34.481+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:52:34.481+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36660,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:52:34.483+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:52:34.484+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:52:34.485+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:52:34.504+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:52:34.504+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:52:34.528+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:52:34.575+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:52:34.575+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:52:35.076+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:52:35.670+0000] [.error] [src/director/director.go:mainLoop:399] [00:52:35.670] Failed to planAndExecute : [00:52:35.670] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:52:35.670] Failed to apply action. Result = : [00:52:35.668] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:52:35.667] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36660 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:52:35.667] Error starting mongod : [00:52:35.666] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36660,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36660 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 40092 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:52:34.481+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:52:34.481+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:52:34.481+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36660,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:52:34.483+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:52:34.484+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:52:34.485+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:52:34.504+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:52:34.504+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:52:34.528+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:52:34.575+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:52:34.575+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:52:35.076+00:00"},"s":"I", "c":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:52:35.738+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:35.738] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:36.546+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:52:36.546] clusterConfig unchanged [2021-10-22T00:52:36.738+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:36.738] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:37.739+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:37.739] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:38.738+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:38.738] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:39.740+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:39.740] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:40.741+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:40.740] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:41.272+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:52:41.272] memory:rss=51609600 alloc=12323240 totalalloc=2836242168 mallocs=36682250 frees=36591205 heapinuse=16105472 heapobjects=91045 gcpausetotalns=1412612120 gcpauselastns=346163 gcnum=454 utimens=49221031000 stimens=15003646000 minflt=135513 majflt=405 [2021-10-22T00:52:41.741+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:41.741] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:42.743+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:42.743] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:43.756+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:43.756] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:44.186+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:52:44.186] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.206222709s [2021-10-22T00:52:44.742+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:44.742] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:45.758+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:45.758] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:46.697+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:52:46.697] clusterConfig unchanged [2021-10-22T00:52:46.743+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:46.743] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:47.744+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:47.744] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:48.746+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:48.746] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:49.746+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:49.746] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:50.747+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:50.746] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:51.747+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:51.747] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:52.751+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:52.751] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:53.750+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:53.750] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:54.751+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:54.751] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:55.752+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:55.752] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:56.752+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:56.752] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:56.828+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:52:56.828] clusterConfig unchanged [2021-10-22T00:52:57.753+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:57.753] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:58.753+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:58.753] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:52:59.754+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:52:59.754] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:00.755+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:00.755] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:01.756+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:01.756] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:02.756+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:02.756] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:03.756+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:03.756] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:04.759+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:04.759] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:05.758+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:05.758] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:06.758+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:06.758] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:06.931+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:53:06.931] clusterConfig unchanged [2021-10-22T00:53:07.761+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:07.761] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:08.759+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:08.759] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:09.760+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:09.760] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:10.760+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:10.760] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:11.761+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:11.761] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:12.762+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:12.762] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:13.762+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:13.762] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:14.763+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:14.763] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:15.769+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:15.769] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:16.765+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:16.765] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:17.052+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:53:17.052] clusterConfig unchanged [2021-10-22T00:53:17.765+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:17.765] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:18.766+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:18.766] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:19.767+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:19.767] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:20.767+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:20.767] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:21.768+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:21.768] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:22.769+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:22.769] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:23.770+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:23.770] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:24.770+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:24.770] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:25.770+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:25.770] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:26.774+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:26.774] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:27.174+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:53:27.174] clusterConfig unchanged [2021-10-22T00:53:27.770+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:27.770] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:28.770+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:28.770] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:29.772+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:29.772] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:30.771+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:30.771] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:31.773+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:31.773] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:32.773+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:32.773] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:33.774+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:33.774] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:34.774+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:34.774] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:35.777+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:35.777] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:35.838+0000] [.info] [src/director/director.go:computePlan:280] [00:53:35.838] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:53:35.838+0000] [.info] [src/director/director.go:tracef:794] [00:53:35.838] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:53:35.838+0000] [.info] [src/director/director.go:tracef:794] [00:53:35.838] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:53:35.838+0000] [.info] [src/director/director.go:tracef:794] [00:53:35.838] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:53:35.838+0000] [.info] [src/director/director.go:tracef:794] [00:53:35.838] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:53:35.838+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:53:35.838] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:53:35.996+0000] [.info] [src/director/director.go:tracef:794] [00:53:35.996] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:53:35.996+0000] [.info] [src/director/director.go:tracef:794] [00:53:35.996] because [2021-10-22T00:53:35.996+0000] [.info] [src/director/director.go:tracef:794] [00:53:35.996] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:53:35.996+0000] [.info] [src/director/director.go:tracef:794] [00:53:35.996] because ['currentState.CanBounceStop' = true] [2021-10-22T00:53:36.153+0000] [.info] [src/director/director.go:tracef:794] [00:53:36.153] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:53:36.153+0000] [.info] [src/director/director.go:tracef:794] [00:53:36.153] because [2021-10-22T00:53:36.153+0000] [.info] [src/director/director.go:tracef:794] [00:53:36.153] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:53:36.153+0000] [.info] [src/director/director.go:tracef:794] [00:53:36.153] because ['currentState.Up' = false] [2021-10-22T00:53:36.153+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:53:36.153] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:53:36.153+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:53:36.153] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:53:36.233+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:53:36.233] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:53:36.241+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:53:36.241] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:53:36.242+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:53:36.242] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:53:36.243+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:53:36.243] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:53:36.244+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:53:36.244] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:53:36.244+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:53:36.244] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:53:36.244+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:53:36.244] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:53:36.245+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:53:36.245] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:53:36.472+0000] [.info] [src/director/director.go:tracef:794] [00:53:36.472] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:53:36.472+0000] [.info] [src/director/director.go:tracef:794] [00:53:36.472] because ['currentState.Up' = false] [2021-10-22T00:53:36.693+0000] [.info] [src/director/director.go:tracef:794] [00:53:36.692] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:53:36.693+0000] [.info] [src/director/director.go:tracef:794] [00:53:36.693] because ['currentState.Up' = false] [2021-10-22T00:53:36.693+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:53:36.693] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:53:36.776+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:36.776] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:36.896+0000] [.info] [src/director/director.go:tracef:794] [00:53:36.896] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:53:36.896+0000] [.info] [src/director/director.go:tracef:794] [00:53:36.896] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:53:37.114+0000] [.info] [src/director/director.go:tracef:794] [00:53:37.114] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:53:37.114+0000] [.info] [src/director/director.go:tracef:794] [00:53:37.114] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:53:37.114+0000] [.info] [src/director/director.go:tracef:794] [00:53:37.114] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:53:37.114+0000] [.info] [src/director/director.go:tracef:794] [00:53:37.114] because ['currentState.Up' = false] [2021-10-22T00:53:37.114+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:53:37.114] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:53:37.114+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:53:37.114] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:53:37.152+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:53:37.152] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:53:37.152+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:53:37.152] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53894 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:53:37.366+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:53:37.366] clusterConfig unchanged [2021-10-22T00:53:37.787+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:37.787] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:37.872+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:53:37.872] ...Finished start [pid=59219] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53894,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:53:37.991+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:53:37.991] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 119.210541ms. [2021-10-22T00:53:37.991+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:53:37.991] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53894 (local=false) [2021-10-22T00:53:37.991+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:53:37.991] Dropping existing documents from system.replset [2021-10-22T00:53:38.144+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:53:38.144] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:53:38.149+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:53:38.149] Inserting oplogTruncateAfterPoint document [2021-10-22T00:53:38.149+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:53:38.149] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:53:38.150+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:53:38.150] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:53:38.150+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:53:38.150] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:53:38.163+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:53:38.163] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:53:38.163+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:53:38.163] Setting the minvalid document to [{_id ObjectID("61720b92523370002becc3b3")} {t -1} {ts {0 1}}] [2021-10-22T00:53:38.178+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:53:38.178] Inserted [{_id ObjectID("61720b92523370002becc3b3")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:53:38.178+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:53:38.178] Inserting new replica set configuration document. [2021-10-22T00:53:38.178+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:53:38.178] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:53:38.191+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:53:38.191] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:53:38.205+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:53:38.205] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:53:38.205+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:53:38.205] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:53:38.205+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:53:38.205] Initiating stopEphemeralMongo [2021-10-22T00:53:38.205+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:53:38.205] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53894 (local=false) [2021-10-22T00:53:38.205+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:53:38.205] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53894 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:53:38.497+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:53:38.497] clusterConfig unchanged [2021-10-22T00:53:38.678+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:53:38.678] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.08707ms. [2021-10-22T00:53:38.678+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:53:38.678] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53894 (local=false) [2021-10-22T00:53:38.678+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:53:38.678] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.526µs. [2021-10-22T00:53:38.678+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:53:38.678] Successfully finished running stopEphemeralMongo [2021-10-22T00:53:38.678+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:53:38.678] Successfully stopped existing ephemeral mongod. [2021-10-22T00:53:38.678+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:53:38.678] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:53:38.678+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:53:38.678] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:53:38.712+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:53:38.712] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:53:38.713+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:53:38.713] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47020 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:53:38.779+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:38.779] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:39.600+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:53:39.600] clusterConfig unchanged [2021-10-22T00:53:39.787+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:39.787] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:39.891+0000] [.error] [src/action/start.go:func1:93] [103] [00:53:39.891] Error starting mongod : [00:53:39.891] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47020,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47020 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:53:38.745Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 60738 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:53:38.766+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:53:38.766+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:53:38.766+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47020,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:53:38.767+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:53:38.769+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:53:38.770+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:53:38.795+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:53:38.795+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:53:38.795+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:53:38.840+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:53:38.841+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:53:39.365+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864019:365364][60738:0x7f0c5a407bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:53:39.468+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864019:468806][60738:0x7f0c5a407bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:53:39.569+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864019:569582][60738:0x7f0c5a407bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T00:53:39.688+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864019:688197][60738:0x7f0c5a407bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:53:39.768+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864019:768374][60738:0x7f0c5a407bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:53:39.840+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864019:840637][60738:0x7f0c5a407bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:53:39.840+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864019:840709][60738:0x7f0c5a407bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:53:39.891+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:53:39.891] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:47020 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:53:39.891] Error starting mongod : [00:53:39.891] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47020,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47020 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:53:38.745Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 60738 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:53:38.766+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:53:38.766+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:53:38.766+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47020,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:53:38.767+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:53:38.769+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:53:38.770+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:53:38.795+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:53:38.795+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:53:38.795+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:53:38.840+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:53:38.841+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:53:39.365+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864019:365364][60738:0x7f0c5a407bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:53:39.468+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:53:39.891+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:53:39.891] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:53:39.891] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:47020 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:53:39.891] Error starting mongod : [00:53:39.891] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47020,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47020 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:53:38.745Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 60738 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:53:38.766+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:53:38.766+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:53:38.766+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47020,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:53:38.767+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:53:38.769+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:53:38.770+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:53:38.795+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:53:38.795+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:53:38.795+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:53:38.840+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:53:38.841+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:53:39.365+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864019:365364][60738:0x7f0c5a407bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:53:39.891+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:53:39.891] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:53:39.891+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:53:39.891] Initiating stopEphemeralMongo [2021-10-22T00:53:39.891+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:53:39.891] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:53:39.892+0000] [.error] [src/director/director.go:executePlan:962] [00:53:39.891] Failed to apply action. Result = : [00:53:39.891] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:53:39.891] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:47020 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:53:39.891] Error starting mongod : [00:53:39.891] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47020,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47020 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:53:38.745Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 60738 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:53:38.766+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:53:38.766+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:53:38.766+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47020,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:53:38.767+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:53:38.769+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:53:38.770+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:53:38.795+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:53:38.795+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:53:38.795+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:53:38.840+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:53:38.841+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:53:39.365+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:53:39.892+0000] [.error] [src/director/director.go:planAndExecute:567] [00:53:39.892] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:53:39.891] Failed to apply action. Result = : [00:53:39.891] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:53:39.891] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:47020 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:53:39.891] Error starting mongod : [00:53:39.891] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47020,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47020 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:53:38.745Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 60738 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:53:38.766+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:53:38.766+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:53:38.766+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47020,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:53:38.767+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:53:38.769+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:53:38.770+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:53:38.795+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:53:38.795+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:53:38.795+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:53:38.840+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:53:38.841+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:53:39.892+0000] [.error] [src/director/director.go:mainLoop:399] [00:53:39.892] Failed to planAndExecute : [00:53:39.892] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:53:39.891] Failed to apply action. Result = : [00:53:39.891] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:53:39.891] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:47020 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:53:39.891] Error starting mongod : [00:53:39.891] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47020,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47020 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:53:38.745Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 60738 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:53:38.766+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:53:38.766+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:53:38.766+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47020,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:53:38.767+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:53:38.769+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:53:38.770+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:53:38.795+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:53:38.795+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:53:38.795+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:53:38.840+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:53:38.841+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:53:40.788+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:40.788] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:40.831+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:53:40.831] clusterConfig unchanged [2021-10-22T00:53:41.273+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:53:41.273] memory:rss=51335168 alloc=12404408 totalalloc=2919392944 mallocs=37740271 frees=37657066 heapinuse=16834560 heapobjects=83205 gcpausetotalns=1416182069 gcpauselastns=96550 gcnum=467 utimens=50802043000 stimens=15574685000 minflt=139584 majflt=405 [2021-10-22T00:53:41.786+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:41.786] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:42.792+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:42.792] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:43.793+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:43.793] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:44.165+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [00:53:44.165] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47020 (local=false) because it is not up [2021-10-22T00:53:44.794+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:44.794] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:45.795+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:45.795] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:46.796+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:46.796] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:47.796+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:47.796] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:48.615+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:53:48.615] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.370110229s [2021-10-22T00:53:48.794+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:48.794] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:49.795+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:49.795] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:50.794+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:50.794] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:50.943+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:53:50.943] clusterConfig unchanged [2021-10-22T00:53:51.795+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:51.795] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:52.796+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:52.796] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:53.797+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:53.797] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:54.798+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:54.798] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:55.798+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:55.798] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:56.798+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:56.798] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:57.799+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:57.799] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:58.800+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:58.800] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:53:59.800+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:53:59.800] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:00.801+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:00.801] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:01.055+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:54:01.055] clusterConfig unchanged [2021-10-22T00:54:01.802+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:01.802] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:02.802+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:02.802] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:03.803+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:03.803] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:04.804+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:04.804] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:05.805+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:05.805] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:06.806+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:06.806] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:07.807+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:07.807] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:08.807+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:08.807] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:09.808+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:09.808] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:10.810+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:10.810] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:11.157+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:54:11.157] clusterConfig unchanged [2021-10-22T00:54:11.815+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:11.815] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:12.810+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:12.810] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:13.811+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:13.811] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:14.811+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:14.811] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:15.815+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:15.815] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:16.812+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:16.812] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:17.813+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:17.813] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:18.814+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:18.814] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:19.815+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:19.815] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:20.816+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:20.816] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:21.259+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:54:21.259] clusterConfig unchanged [2021-10-22T00:54:21.816+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:21.816] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:22.823+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:22.823] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:23.817+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:23.817] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:24.817+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:24.817] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:25.819+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:25.819] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:26.818+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:26.818] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:27.819+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:27.819] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:28.821+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:28.821] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:29.821+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:29.821] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:30.821+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:30.821] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:31.371+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:54:31.371] clusterConfig unchanged [2021-10-22T00:54:31.822+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:31.822] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:32.822+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:32.822] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:33.829+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:33.829] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:34.823+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:34.823] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:35.825+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:35.825] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:36.825+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:36.825] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:37.826+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:37.826] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:38.826+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:38.826] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:39.826+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:39.826] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:40.058+0000] [.info] [src/director/director.go:computePlan:280] [00:54:40.058] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:54:40.058+0000] [.info] [src/director/director.go:tracef:794] [00:54:40.058] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:54:40.058+0000] [.info] [src/director/director.go:tracef:794] [00:54:40.058] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:54:40.058+0000] [.info] [src/director/director.go:tracef:794] [00:54:40.058] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:54:40.058+0000] [.info] [src/director/director.go:tracef:794] [00:54:40.058] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:54:40.058+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:54:40.058] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:54:40.223+0000] [.info] [src/director/director.go:tracef:794] [00:54:40.223] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:54:40.223+0000] [.info] [src/director/director.go:tracef:794] [00:54:40.223] because [2021-10-22T00:54:40.223+0000] [.info] [src/director/director.go:tracef:794] [00:54:40.223] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:54:40.223+0000] [.info] [src/director/director.go:tracef:794] [00:54:40.223] because ['currentState.CanBounceStop' = true] [2021-10-22T00:54:40.374+0000] [.info] [src/director/director.go:tracef:794] [00:54:40.374] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:54:40.374+0000] [.info] [src/director/director.go:tracef:794] [00:54:40.374] because [2021-10-22T00:54:40.374+0000] [.info] [src/director/director.go:tracef:794] [00:54:40.374] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:54:40.374+0000] [.info] [src/director/director.go:tracef:794] [00:54:40.374] because ['currentState.Up' = false] [2021-10-22T00:54:40.374+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:54:40.374] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:54:40.374+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:54:40.374] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:54:40.560+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:54:40.560] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:54:40.568+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:54:40.568] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:54:40.568+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:54:40.568] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:54:40.569+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:54:40.569] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:54:40.570+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:54:40.570] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:54:40.570+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:54:40.570] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:54:40.570+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:54:40.570] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:54:40.572+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:54:40.572] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:54:40.812+0000] [.info] [src/director/director.go:tracef:794] [00:54:40.812] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:54:40.812+0000] [.info] [src/director/director.go:tracef:794] [00:54:40.812] because ['currentState.Up' = false] [2021-10-22T00:54:40.830+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:40.830] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:41.038+0000] [.info] [src/director/director.go:tracef:794] [00:54:41.038] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:54:41.038+0000] [.info] [src/director/director.go:tracef:794] [00:54:41.038] because ['currentState.Up' = false] [2021-10-22T00:54:41.038+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:54:41.038] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:54:41.236+0000] [.info] [src/director/director.go:tracef:794] [00:54:41.236] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:54:41.236+0000] [.info] [src/director/director.go:tracef:794] [00:54:41.236] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:54:41.283+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:54:41.283] memory:rss=50712576 alloc=8650616 totalalloc=2989814288 mallocs=38658011 frees=38586539 heapinuse=13377536 heapobjects=71472 gcpausetotalns=1417192187 gcpauselastns=102155 gcnum=479 utimens=52083853000 stimens=16008506000 minflt=143517 majflt=405 [2021-10-22T00:54:41.442+0000] [.info] [src/director/director.go:tracef:794] [00:54:41.442] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:54:41.442+0000] [.info] [src/director/director.go:tracef:794] [00:54:41.442] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:54:41.443+0000] [.info] [src/director/director.go:tracef:794] [00:54:41.443] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:54:41.443+0000] [.info] [src/director/director.go:tracef:794] [00:54:41.443] because ['currentState.Up' = false] [2021-10-22T00:54:41.443+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:54:41.443] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:54:41.443+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:54:41.443] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:54:41.479+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:54:41.479] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:54:41.480+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:54:41.480] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34131 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:54:41.582+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:54:41.582] clusterConfig unchanged [2021-10-22T00:54:41.840+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:41.840] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:42.222+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:54:42.222] ...Finished start [pid=5924] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34131,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:54:42.333+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:54:42.333] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 110.88101ms. [2021-10-22T00:54:42.333+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:54:42.333] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34131 (local=false) [2021-10-22T00:54:42.333+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:54:42.333] Dropping existing documents from system.replset [2021-10-22T00:54:42.473+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:54:42.473] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:54:42.477+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:54:42.477] Inserting oplogTruncateAfterPoint document [2021-10-22T00:54:42.477+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:54:42.477] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:54:42.478+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:54:42.478] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:54:42.478+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:54:42.478] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:54:42.491+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:54:42.491] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:54:42.491+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:54:42.491] Setting the minvalid document to [{_id ObjectID("61720bd2523370002becc3c1")} {t -1} {ts {0 1}}] [2021-10-22T00:54:42.504+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:54:42.504] Inserted [{_id ObjectID("61720bd2523370002becc3c1")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:54:42.504+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:54:42.504] Inserting new replica set configuration document. [2021-10-22T00:54:42.505+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:54:42.505] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:54:42.517+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:54:42.517] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:54:42.521+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:54:42.521] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:54:42.521+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:54:42.521] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:54:42.521+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:54:42.521] Initiating stopEphemeralMongo [2021-10-22T00:54:42.521+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:54:42.521] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34131 (local=false) [2021-10-22T00:54:42.521+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:54:42.521] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34131 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:54:42.728+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:54:42.728] clusterConfig unchanged [2021-10-22T00:54:42.837+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:42.837] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:42.867+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:54:42.867] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.879071ms. [2021-10-22T00:54:42.867+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:54:42.867] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34131 (local=false) [2021-10-22T00:54:42.867+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:54:42.867] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.471µs. [2021-10-22T00:54:42.867+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:54:42.867] Successfully finished running stopEphemeralMongo [2021-10-22T00:54:42.867+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:54:42.867] Successfully stopped existing ephemeral mongod. [2021-10-22T00:54:42.867+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:54:42.867] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:54:42.867+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:54:42.867] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:54:42.903+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:54:42.903] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:54:42.904+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:54:42.903] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34710 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:54:43.842+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:43.842] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:43.946+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:54:43.946] clusterConfig unchanged [2021-10-22T00:54:44.052+0000] [.error] [src/action/start.go:func1:93] [103] [00:54:44.052] Error starting mongod : [00:54:44.052] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34710,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34710 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 7504 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: h1p-config-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:54:42.953+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:54:42.953+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:54:42.953+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34710,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:54:42.954+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:54:42.955+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:54:42.956+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:54:43.016+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:54:43.016+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:54:43.027+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:54:43.073+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:54:43.073+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:54:43.552+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864083:552372][7504:0x7f384422abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:54:43.655+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864083:654987][7504:0x7f384422abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:54:43.757+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864083:757423][7504:0x7f384422abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T00:54:43.869+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864083:869254][7504:0x7f384422abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:54:43.942+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864083:942484][7504:0x7f384422abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:54:44.000+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864084:832][7504:0x7f384422abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:54:44.000+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864084:903][7504:0x7f384422abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:54:44.002+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"i...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:54:44.052+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:54:44.052] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34710 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:54:44.052] Error starting mongod : [00:54:44.052] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34710,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34710 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 7504 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: h1p-config-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:54:42.953+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:54:42.953+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:54:42.953+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34710,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:54:42.954+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:54:42.955+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:54:42.956+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:54:43.016+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:54:43.016+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:54:43.027+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:54:43.073+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:54:43.073+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:54:43.552+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864083:552372][7504:0x7f384422abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:54:43.655+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864083:654987][7504:0x7f384422abc0], txn...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:54:44.052+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:54:44.052] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:54:44.052] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34710 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:54:44.052] Error starting mongod : [00:54:44.052] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34710,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34710 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 7504 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: h1p-config-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:54:42.953+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:54:42.953+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:54:42.953+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34710,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:54:42.954+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:54:42.955+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:54:42.956+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:54:43.016+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:54:43.016+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:54:43.027+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:54:43.073+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:54:43.073+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:54:43.552+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864083:552372][7504:0x7f384422abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:54:43.655+00:00"},"s":"I", "c":"STORAGE", "id"... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:54:44.053+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:54:44.053] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:54:44.053+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:54:44.053] Initiating stopEphemeralMongo [2021-10-22T00:54:44.053+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:54:44.053] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:54:44.053+0000] [.error] [src/director/director.go:executePlan:962] [00:54:44.053] Failed to apply action. Result = : [00:54:44.052] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:54:44.052] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34710 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:54:44.052] Error starting mongod : [00:54:44.052] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34710,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34710 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 7504 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: h1p-config-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:54:42.953+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:54:42.953+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:54:42.953+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34710,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:54:42.954+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:54:42.955+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:54:42.956+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:54:43.016+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:54:43.016+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:54:43.027+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:54:43.073+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:54:43.073+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:54:43.552+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864083:552372][7504:0x7f384422abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 thr... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:54:44.053+0000] [.error] [src/director/director.go:planAndExecute:567] [00:54:44.053] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:54:44.053] Failed to apply action. Result = : [00:54:44.052] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:54:44.052] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34710 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:54:44.052] Error starting mongod : [00:54:44.052] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34710,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34710 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 7504 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: h1p-config-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:54:42.953+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:54:42.953+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:54:42.953+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34710,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:54:42.954+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:54:42.955+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:54:42.956+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:54:43.016+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:54:43.016+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:54:43.027+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:54:43.073+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:54:43.073+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:54:43.552+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"Wire... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:54:44.053+0000] [.error] [src/director/director.go:mainLoop:399] [00:54:44.053] Failed to planAndExecute : [00:54:44.053] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:54:44.053] Failed to apply action. Result = : [00:54:44.052] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:54:44.052] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34710 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:54:44.052] Error starting mongod : [00:54:44.052] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34710,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34710 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 7504 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: h1p-config-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:54:42.953+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:54:42.953+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:54:42.953+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34710,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:54:42.954+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:54:42.955+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:54:42.956+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:54:43.016+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:54:43.016+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:54:43.027+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:54:43.073+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:54:43.073+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:54:43.552+00:00"},... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:54:44.841+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:44.841] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:45.195+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:54:45.195] clusterConfig unchanged [2021-10-22T00:54:45.843+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:45.843] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:46.843+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:46.843] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:47.842+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:47.842] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:48.843+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:48.843] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:49.846+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:49.846] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:50.849+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:50.849] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:51.848+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:51.848] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:52.740+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:54:52.740] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.168786274s [2021-10-22T00:54:52.846+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:52.846] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:53.846+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:53.846] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:54.846+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:54.846] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:55.337+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:54:55.337] clusterConfig unchanged [2021-10-22T00:54:55.846+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:55.846] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:56.847+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:56.847] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:57.848+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:57.848] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:58.848+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:58.848] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:54:59.849+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:54:59.849] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:00.849+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:00.849] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:01.855+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:01.855] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:02.850+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:02.850] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:03.850+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:03.850] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:04.851+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:04.851] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:05.429+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:55:05.429] clusterConfig unchanged [2021-10-22T00:55:05.852+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:05.852] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:06.852+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:06.852] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:07.852+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:07.852] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:08.853+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:08.853] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:09.853+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:09.853] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:10.853+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:10.853] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:11.854+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:11.854] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:12.854+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:12.854] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:13.855+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:13.855] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:14.855+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:14.855] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:15.591+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:55:15.591] clusterConfig unchanged [2021-10-22T00:55:15.856+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:15.856] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:16.855+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:16.855] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:17.856+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:17.856] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:18.857+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:18.857] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:19.857+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:19.857] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:20.857+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:20.857] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:21.858+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:21.858] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:22.859+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:22.859] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:23.867+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:23.867] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:24.860+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:24.860] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:25.722+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:55:25.722] clusterConfig unchanged [2021-10-22T00:55:25.861+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:25.861] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:26.861+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:26.861] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:27.862+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:27.862] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:28.862+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:28.862] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:29.863+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:29.863] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:30.864+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:30.864] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:31.864+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:31.864] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:32.864+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:32.864] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:33.865+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:33.865] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:34.865+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:34.865] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:35.873+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:35.873] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:35.874+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:55:35.874] clusterConfig unchanged [2021-10-22T00:55:36.867+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:36.867] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:37.867+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:37.867] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:38.869+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:38.869] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:39.869+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:39.869] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:40.869+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:40.869] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:41.284+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:55:41.284] memory:rss=50212864 alloc=9662568 totalalloc=3038202960 mallocs=39210046 frees=39136669 heapinuse=14082048 heapobjects=73377 gcpausetotalns=1417922278 gcpauselastns=90043 gcnum=487 utimens=53160373000 stimens=16479070000 minflt=146947 majflt=405 [2021-10-22T00:55:41.869+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:41.869] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:42.870+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:42.870] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:43.875+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:43.875] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:44.218+0000] [.info] [src/director/director.go:computePlan:280] [00:55:44.218] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:55:44.218+0000] [.info] [src/director/director.go:tracef:794] [00:55:44.218] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:55:44.218+0000] [.info] [src/director/director.go:tracef:794] [00:55:44.218] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:55:44.218+0000] [.info] [src/director/director.go:tracef:794] [00:55:44.218] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:55:44.218+0000] [.info] [src/director/director.go:tracef:794] [00:55:44.218] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:55:44.218+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:55:44.218] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:55:44.371+0000] [.info] [src/director/director.go:tracef:794] [00:55:44.371] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:55:44.371+0000] [.info] [src/director/director.go:tracef:794] [00:55:44.371] because [2021-10-22T00:55:44.371+0000] [.info] [src/director/director.go:tracef:794] [00:55:44.371] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:55:44.371+0000] [.info] [src/director/director.go:tracef:794] [00:55:44.371] because ['currentState.CanBounceStop' = true] [2021-10-22T00:55:44.517+0000] [.info] [src/director/director.go:tracef:794] [00:55:44.517] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:55:44.517+0000] [.info] [src/director/director.go:tracef:794] [00:55:44.517] because [2021-10-22T00:55:44.517+0000] [.info] [src/director/director.go:tracef:794] [00:55:44.517] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:55:44.517+0000] [.info] [src/director/director.go:tracef:794] [00:55:44.517] because ['currentState.Up' = false] [2021-10-22T00:55:44.518+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:55:44.518] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:55:44.518+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:55:44.518] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:55:44.597+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:55:44.597] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:55:44.606+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:55:44.606] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:55:44.607+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:55:44.607] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:55:44.610+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:55:44.610] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:55:44.610+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:55:44.610] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:55:44.614+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:55:44.614] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:55:44.615+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:55:44.615] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:55:44.818+0000] [.info] [src/director/director.go:tracef:794] [00:55:44.818] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:55:44.818+0000] [.info] [src/director/director.go:tracef:794] [00:55:44.818] because ['currentState.Up' = false] [2021-10-22T00:55:44.892+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:44.892] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:45.035+0000] [.info] [src/director/director.go:tracef:794] [00:55:45.035] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:55:45.035+0000] [.info] [src/director/director.go:tracef:794] [00:55:45.035] because ['currentState.Up' = false] [2021-10-22T00:55:45.035+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:55:45.035] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:55:45.245+0000] [.info] [src/director/director.go:tracef:794] [00:55:45.245] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:55:45.245+0000] [.info] [src/director/director.go:tracef:794] [00:55:45.245] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:55:45.459+0000] [.info] [src/director/director.go:tracef:794] [00:55:45.459] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:55:45.459+0000] [.info] [src/director/director.go:tracef:794] [00:55:45.459] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:55:45.459+0000] [.info] [src/director/director.go:tracef:794] [00:55:45.459] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:55:45.459+0000] [.info] [src/director/director.go:tracef:794] [00:55:45.459] because ['currentState.Up' = false] [2021-10-22T00:55:45.459+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:55:45.459] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:55:45.459+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:55:45.459] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:55:45.491+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:55:45.491] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:55:45.492+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:55:45.492] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38779 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:55:45.885+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:45.885] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:46.095+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:55:46.095] clusterConfig unchanged [2021-10-22T00:55:46.214+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:55:46.214] ...Finished start [pid=26615] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38779,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:55:46.336+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:55:46.336] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 121.816117ms. [2021-10-22T00:55:46.336+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:55:46.336] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38779 (local=false) [2021-10-22T00:55:46.336+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:55:46.336] Dropping existing documents from system.replset [2021-10-22T00:55:46.473+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:55:46.473] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:55:46.491+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:55:46.490] Inserting oplogTruncateAfterPoint document [2021-10-22T00:55:46.491+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:55:46.491] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:55:46.492+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:55:46.492] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:55:46.492+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:55:46.492] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:55:46.506+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:55:46.505] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:55:46.506+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:55:46.506] Setting the minvalid document to [{_id ObjectID("61720c12523370002becc3cf")} {t -1} {ts {0 1}}] [2021-10-22T00:55:46.517+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:55:46.517] Inserted [{_id ObjectID("61720c12523370002becc3cf")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:55:46.517+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:55:46.517] Inserting new replica set configuration document. [2021-10-22T00:55:46.517+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:55:46.517] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:55:46.530+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:55:46.530] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:55:46.546+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:55:46.546] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:55:46.546+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:55:46.546] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:55:46.546+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:55:46.546] Initiating stopEphemeralMongo [2021-10-22T00:55:46.546+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:55:46.546] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38779 (local=false) [2021-10-22T00:55:46.546+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:55:46.546] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38779 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:55:46.884+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:46.884] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:46.894+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:55:46.894] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.347811ms. [2021-10-22T00:55:46.894+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:55:46.894] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38779 (local=false) [2021-10-22T00:55:46.894+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:55:46.894] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 38.165µs. [2021-10-22T00:55:46.894+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:55:46.894] Successfully finished running stopEphemeralMongo [2021-10-22T00:55:46.894+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:55:46.894] Successfully stopped existing ephemeral mongod. [2021-10-22T00:55:46.894+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:55:46.894] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:55:46.894+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:55:46.894] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:55:46.930+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:55:46.930] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:55:46.931+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:55:46.931] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38873 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:55:47.247+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:55:47.247] clusterConfig unchanged [2021-10-22T00:55:47.933+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:47.933] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:48.125+0000] [.error] [src/action/start.go:func1:93] [103] [00:55:48.125] Error starting mongod : [00:55:48.125] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38873 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:55:46.962Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 28169 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:55:46.982+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:55:46.982+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:55:46.982+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:55:46.983+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:55:46.985+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:55:46.985+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:55:47.016+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:55:47.016+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:55:47.016+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:55:47.062+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:55:47.062+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:55:47.621+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864147:621314][28169:0x7f9ca51d6bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:55:47.724+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864147:724932][28169:0x7f9ca51d6bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:55:47.825+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864147:825271][28169:0x7f9ca51d6bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T00:55:47.935+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864147:935376][28169:0x7f9ca51d6bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:55:48.006+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864148:6092][28169:0x7f9ca51d6bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:55:48.075+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864148:75682][28169:0x7f9ca51d6bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:55:48.075+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864148:75773][28169:0x7f9ca51d6bc0], txn-recover: [WT_VERB_RECOVERY | WT_...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:55:48.125+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:55:48.125] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38873 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:55:48.125] Error starting mongod : [00:55:48.125] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38873 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:55:46.962Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 28169 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:55:46.982+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:55:46.982+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:55:46.982+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:55:46.983+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:55:46.985+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:55:46.985+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:55:47.016+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:55:47.016+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:55:47.016+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:55:47.062+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:55:47.062+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:55:47.621+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864147:621314][28169:0x7f9ca51d6bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:55:47.724+00:00"},"s"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:55:48.126+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:55:48.126] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:55:48.125] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38873 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:55:48.125] Error starting mongod : [00:55:48.125] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38873 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:55:46.962Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 28169 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:55:46.982+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:55:46.982+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:55:46.982+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:55:46.983+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:55:46.985+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:55:46.985+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:55:47.016+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:55:47.016+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:55:47.016+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:55:47.062+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:55:47.062+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:55:47.621+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864147:621314][28169:0x7f9ca51d6bc0]... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:55:48.126+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:55:48.126] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:55:48.126+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:55:48.126] Initiating stopEphemeralMongo [2021-10-22T00:55:48.126+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:55:48.126] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:55:48.127+0000] [.error] [src/director/director.go:executePlan:962] [00:55:48.127] Failed to apply action. Result = : [00:55:48.126] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:55:48.125] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38873 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:55:48.125] Error starting mongod : [00:55:48.125] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38873 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:55:46.962Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 28169 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:55:46.982+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:55:46.982+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:55:46.982+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:55:46.983+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:55:46.985+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:55:46.985+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:55:47.016+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:55:47.016+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:55:47.016+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:55:47.062+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:55:47.062+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:55:47.621+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten",... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:55:48.127+0000] [.error] [src/director/director.go:planAndExecute:567] [00:55:48.127] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:55:48.127] Failed to apply action. Result = : [00:55:48.126] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:55:48.125] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38873 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:55:48.125] Error starting mongod : [00:55:48.125] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38873 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:55:46.962Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 28169 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:55:46.982+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:55:46.982+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:55:46.982+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:55:46.983+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:55:46.985+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:55:46.985+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:55:47.016+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:55:47.016+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:55:47.016+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:55:47.062+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:55:47.062+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:55:48.127+0000] [.error] [src/director/director.go:mainLoop:399] [00:55:48.127] Failed to planAndExecute : [00:55:48.127] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:55:48.127] Failed to apply action. Result = : [00:55:48.126] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:55:48.125] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38873 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:55:48.125] Error starting mongod : [00:55:48.125] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38873 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:55:46.962Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 28169 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:55:46.982+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:55:46.982+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:55:46.982+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:55:46.983+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:55:46.985+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:55:46.985+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:55:47.016+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:55:47.016+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:55:47.016+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:55:47.062+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:55:47.062+00:00"},"s":"I", "c":"NETWORK", "id":5... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:55:48.565+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:55:48.565] clusterConfig unchanged [2021-10-22T00:55:48.888+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:48.888] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:49.887+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:49.887] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:50.888+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:50.888] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:51.887+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:51.887] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:52.889+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:52.889] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:53.901+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:53.901] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:54.892+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:54.892] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:55.891+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:55.891] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:56.825+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:55:56.825] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.209431944s [2021-10-22T00:55:56.888+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:56.888] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:57.888+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:57.888] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:58.707+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:55:58.707] clusterConfig unchanged [2021-10-22T00:55:58.889+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:58.889] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:55:59.890+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:55:59.890] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:00.892+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:00.892] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:01.892+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:01.892] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:02.892+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:02.892] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:03.895+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:03.895] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:04.893+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:04.893] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:05.900+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:05.900] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:06.895+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:06.895] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:07.911+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:07.911] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:08.829+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:56:08.829] clusterConfig unchanged [2021-10-22T00:56:08.897+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:08.897] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:09.898+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:09.898] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:10.898+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:10.898] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:11.898+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:11.898] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:12.898+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:12.898] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:13.899+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:13.899] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:14.899+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:14.899] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:15.901+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:15.901] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:16.901+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:16.901] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:17.901+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:17.901] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:18.557+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [00:56:18.556] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T00:56:18.559+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:18.559] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T00:56:18.559+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [00:56:18.559] Could not get version : [00:56:18.559] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T00:56:18.559+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [00:56:18.559] Error getting version for log lines. Skipping : [00:56:18.559] Could not get version : [00:56:18.559] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T00:56:18.902+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:18.902] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:18.935+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:56:18.935] clusterConfig unchanged [2021-10-22T00:56:19.904+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:19.904] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:20.904+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:20.904] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:21.902+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:21.902] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:22.904+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:22.904] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:23.904+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:23.904] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:24.904+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:24.904] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:25.911+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:25.911] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:26.906+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:26.906] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:27.906+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:27.906] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:28.906+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:28.906] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:29.033+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:56:29.033] clusterConfig unchanged [2021-10-22T00:56:29.907+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:29.907] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:30.908+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:30.908] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:31.908+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:31.908] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:32.909+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:32.909] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:33.910+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:33.910] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:34.911+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:34.911] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:35.912+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:35.912] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:36.912+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:36.912] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:37.918+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:37.918] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:38.913+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:38.913] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:39.145+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:56:39.145] clusterConfig unchanged [2021-10-22T00:56:39.914+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:39.914] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:40.917+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:40.917] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:41.285+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:56:41.285] memory:rss=50393088 alloc=8557288 totalalloc=3125063264 mallocs=40291079 frees=40229569 heapinuse=14000128 heapobjects=61510 gcpausetotalns=1420811270 gcpauselastns=153241 gcnum=501 utimens=54752394000 stimens=17049241000 minflt=151796 majflt=405 [2021-10-22T00:56:41.915+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:41.915] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:42.916+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:42.916] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:43.916+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:43.916] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:44.173+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [00:56:44.173] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38873 (local=false) because it is not up [2021-10-22T00:56:44.920+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:44.920] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:45.918+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:45.918] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:46.918+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:46.918] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:47.918+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:47.918] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:48.290+0000] [.info] [src/director/director.go:computePlan:280] [00:56:48.290] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:56:48.290+0000] [.info] [src/director/director.go:tracef:794] [00:56:48.290] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:56:48.290+0000] [.info] [src/director/director.go:tracef:794] [00:56:48.290] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:56:48.291+0000] [.info] [src/director/director.go:tracef:794] [00:56:48.291] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:56:48.291+0000] [.info] [src/director/director.go:tracef:794] [00:56:48.291] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:56:48.291+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:56:48.291] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:56:48.454+0000] [.info] [src/director/director.go:tracef:794] [00:56:48.454] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:56:48.455+0000] [.info] [src/director/director.go:tracef:794] [00:56:48.454] because [2021-10-22T00:56:48.455+0000] [.info] [src/director/director.go:tracef:794] [00:56:48.455] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:56:48.455+0000] [.info] [src/director/director.go:tracef:794] [00:56:48.455] because ['currentState.CanBounceStop' = true] [2021-10-22T00:56:48.606+0000] [.info] [src/director/director.go:tracef:794] [00:56:48.606] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:56:48.606+0000] [.info] [src/director/director.go:tracef:794] [00:56:48.606] because [2021-10-22T00:56:48.606+0000] [.info] [src/director/director.go:tracef:794] [00:56:48.606] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:56:48.607+0000] [.info] [src/director/director.go:tracef:794] [00:56:48.606] because ['currentState.Up' = false] [2021-10-22T00:56:48.607+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:56:48.607] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:56:48.607+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:56:48.607] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:56:48.765+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:56:48.765] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:56:48.775+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:56:48.775] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:56:48.776+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:56:48.776] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:56:48.777+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:56:48.777] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:56:48.778+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:56:48.778] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:56:48.778+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:56:48.778] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:56:48.780+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:56:48.780] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:56:48.943+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:48.943] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:48.994+0000] [.info] [src/director/director.go:tracef:794] [00:56:48.994] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:56:48.994+0000] [.info] [src/director/director.go:tracef:794] [00:56:48.994] because ['currentState.Up' = false] [2021-10-22T00:56:49.190+0000] [.info] [src/director/director.go:tracef:794] [00:56:49.190] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:56:49.190+0000] [.info] [src/director/director.go:tracef:794] [00:56:49.190] because ['currentState.Up' = false] [2021-10-22T00:56:49.190+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:56:49.190] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:56:49.369+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:56:49.369] clusterConfig unchanged [2021-10-22T00:56:49.399+0000] [.info] [src/director/director.go:tracef:794] [00:56:49.399] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:56:49.399+0000] [.info] [src/director/director.go:tracef:794] [00:56:49.399] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:56:49.610+0000] [.info] [src/director/director.go:tracef:794] [00:56:49.610] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:56:49.610+0000] [.info] [src/director/director.go:tracef:794] [00:56:49.610] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:56:49.610+0000] [.info] [src/director/director.go:tracef:794] [00:56:49.610] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:56:49.610+0000] [.info] [src/director/director.go:tracef:794] [00:56:49.610] because ['currentState.Up' = false] [2021-10-22T00:56:49.610+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:56:49.610] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:56:49.610+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:56:49.610] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:56:49.644+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:56:49.644] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:56:49.644+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:56:49.644] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45378 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:56:49.929+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:49.929] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:50.362+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:56:50.362] ...Finished start [pid=47376] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45378,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:56:50.481+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:56:50.481] clusterConfig unchanged [2021-10-22T00:56:50.494+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:56:50.494] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 131.79414ms. [2021-10-22T00:56:50.494+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:56:50.494] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45378 (local=false) [2021-10-22T00:56:50.494+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:56:50.494] Dropping existing documents from system.replset [2021-10-22T00:56:50.656+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:56:50.656] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:56:50.690+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:56:50.690] Inserting oplogTruncateAfterPoint document [2021-10-22T00:56:50.690+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:56:50.690] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:56:50.692+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:56:50.692] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:56:50.692+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:56:50.692] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:56:50.705+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:56:50.705] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:56:50.705+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:56:50.705] Setting the minvalid document to [{_id ObjectID("61720c52523370002becc3dd")} {t -1} {ts {0 1}}] [2021-10-22T00:56:50.719+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:56:50.719] Inserted [{_id ObjectID("61720c52523370002becc3dd")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:56:50.719+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:56:50.719] Inserting new replica set configuration document. [2021-10-22T00:56:50.719+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:56:50.719] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:56:50.732+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:56:50.732] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:56:50.766+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:56:50.766] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:56:50.766+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:56:50.766] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:56:50.766+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:56:50.766] Initiating stopEphemeralMongo [2021-10-22T00:56:50.766+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:56:50.766] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45378 (local=false) [2021-10-22T00:56:50.766+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:56:50.766] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45378 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:56:50.939+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:50.939] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:51.179+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:56:51.179] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.169977ms. [2021-10-22T00:56:51.179+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:56:51.179] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45378 (local=false) [2021-10-22T00:56:51.180+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:56:51.180] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 110.834µs. [2021-10-22T00:56:51.180+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:56:51.180] Successfully finished running stopEphemeralMongo [2021-10-22T00:56:51.180+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:56:51.180] Successfully stopped existing ephemeral mongod. [2021-10-22T00:56:51.180+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:56:51.180] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:56:51.180+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:56:51.180] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:56:51.223+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:56:51.223] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:56:51.223+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:56:51.223] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53114 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:56:51.590+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:56:51.590] clusterConfig unchanged [2021-10-22T00:56:51.939+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:51.939] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:52.428+0000] [.error] [src/action/start.go:func1:93] [103] [00:56:52.428] Error starting mongod : [00:56:52.428] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53114 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:56:51.257Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 49014 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:56:51.304+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:56:51.304+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:56:51.304+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:56:51.306+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:56:51.307+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:56:51.310+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:56:51.321+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:56:51.321+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:56:51.322+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:56:51.367+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:56:51.367+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:56:51.913+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864211:913947][49014:0x7fab42519bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:56:52.017+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864212:17857][49014:0x7fab42519bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:56:52.117+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864212:117862][49014:0x7fab42519bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T00:56:52.234+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864212:234257][49014:0x7fab42519bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:56:52.310+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864212:310249][49014:0x7fab42519bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:56:52.375+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864212:375301][49014:0x7fab42519bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:56:52.375+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864212:375373][49014:0x7fab42519bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:56:52.428+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:56:52.428] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53114 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:56:52.428] Error starting mongod : [00:56:52.428] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53114 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:56:51.257Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 49014 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:56:51.304+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:56:51.304+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:56:51.304+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:56:51.306+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:56:51.307+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:56:51.310+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:56:51.321+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:56:51.321+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:56:51.322+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:56:51.367+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:56:51.367+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:56:51.913+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864211:913947][49014:0x7fab42519bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:56:52.017+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:56:52.428+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:56:52.428] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:56:52.428] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53114 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:56:52.428] Error starting mongod : [00:56:52.428] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53114 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:56:51.257Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 49014 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:56:51.304+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:56:51.304+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:56:51.304+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:56:51.306+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:56:51.307+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:56:51.310+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:56:51.321+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:56:51.321+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:56:51.322+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:56:51.367+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:56:51.367+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:56:51.913+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864211:913947][49014:0x7fab42519bc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:56:52.429+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:56:52.429] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:56:52.429+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:56:52.429] Initiating stopEphemeralMongo [2021-10-22T00:56:52.429+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:56:52.429] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:56:52.429+0000] [.error] [src/director/director.go:executePlan:962] [00:56:52.429] Failed to apply action. Result = : [00:56:52.428] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:56:52.428] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53114 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:56:52.428] Error starting mongod : [00:56:52.428] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53114 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:56:51.257Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 49014 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:56:51.304+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:56:51.304+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:56:51.304+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:56:51.306+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:56:51.307+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:56:51.310+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:56:51.321+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:56:51.321+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:56:51.322+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:56:51.367+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:56:51.367+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:56:51.913+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:56:52.429+0000] [.error] [src/director/director.go:planAndExecute:567] [00:56:52.429] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:56:52.429] Failed to apply action. Result = : [00:56:52.428] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:56:52.428] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53114 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:56:52.428] Error starting mongod : [00:56:52.428] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53114 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:56:51.257Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 49014 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:56:51.304+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:56:51.304+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:56:51.304+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:56:51.306+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:56:51.307+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:56:51.310+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:56:51.321+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:56:51.321+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:56:51.322+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:56:51.367+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:56:51.367+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:56:52.429+0000] [.error] [src/director/director.go:mainLoop:399] [00:56:52.429] Failed to planAndExecute : [00:56:52.429] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:56:52.429] Failed to apply action. Result = : [00:56:52.428] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:56:52.428] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53114 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:56:52.428] Error starting mongod : [00:56:52.428] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53114 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:56:51.257Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 49014 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:56:51.304+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:56:51.304+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:56:51.304+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:56:51.306+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:56:51.307+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:56:51.310+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:56:51.321+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:56:51.321+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:56:51.322+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:56:51.367+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:56:51.367+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:56:52.850+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:56:52.850] clusterConfig unchanged [2021-10-22T00:56:52.939+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:52.939] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:53.939+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:53.939] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:54.939+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:54.939] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:55.940+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:55.940] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:56.941+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:56.941] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:57.941+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:57.941] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:58.939+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:58.939] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:56:59.940+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:56:59.940] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:00.942+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:00.942] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:01.111+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:57:01.111] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.331650613s [2021-10-22T00:57:01.940+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:01.940] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:02.940+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:02.940] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:02.952+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:57:02.952] clusterConfig unchanged [2021-10-22T00:57:03.946+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:03.946] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:04.941+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:04.941] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:05.942+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:05.942] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:06.943+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:06.943] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:07.943+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:07.943] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:08.943+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:08.943] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:09.947+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:09.947] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:10.945+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:10.945] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:11.945+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:11.945] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:12.946+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:12.945] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:13.044+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:57:13.044] clusterConfig unchanged [2021-10-22T00:57:13.948+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:13.948] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:14.947+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:14.947] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:15.953+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:15.953] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:16.948+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:16.948] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:17.948+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:17.948] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:18.949+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:18.949] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:19.950+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:19.950] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:20.950+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:20.950] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:21.951+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:21.951] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:22.952+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:22.952] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:23.166+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:57:23.166] clusterConfig unchanged [2021-10-22T00:57:23.953+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:23.953] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:24.953+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:24.953] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:25.959+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:25.959] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:26.958+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:26.958] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:27.558+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [00:57:27.558] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T00:57:27.561+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [00:57:27.561] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T00:57:27.954+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:27.954] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:28.955+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:28.955] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:29.956+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:29.956] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:30.957+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:30.957] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:31.958+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:31.958] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:32.957+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:32.957] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:33.288+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:57:33.288] clusterConfig unchanged [2021-10-22T00:57:33.958+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:33.958] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:34.960+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [00:57:34.960] lsblk cache invalidated, running lsblk command [2021-10-22T00:57:34.962+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:34.962] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:35.961+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:35.961] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:36.960+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:36.960] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:37.961+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:37.961] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:38.967+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:38.967] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:39.961+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:39.961] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:40.962+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:40.962] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:41.286+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:57:41.286] memory:rss=50499584 alloc=8071648 totalalloc=3211727656 mallocs=41377164 frees=41321416 heapinuse=13860864 heapobjects=55748 gcpausetotalns=1422384683 gcpauselastns=94732 gcnum=515 utimens=56377604000 stimens=17626036000 minflt=155796 majflt=405 [2021-10-22T00:57:41.963+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:41.963] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:42.963+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:42.963] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:43.410+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:57:43.410] clusterConfig unchanged [2021-10-22T00:57:43.964+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:43.964] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:44.965+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:44.965] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:45.966+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:45.966] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:46.966+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:46.966] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:47.973+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:47.973] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:48.968+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:48.968] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:49.968+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:49.968] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:50.969+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:50.969] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:51.971+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:51.971] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:52.597+0000] [.info] [src/director/director.go:computePlan:280] [00:57:52.597] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:57:52.597+0000] [.info] [src/director/director.go:tracef:794] [00:57:52.597] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:57:52.597+0000] [.info] [src/director/director.go:tracef:794] [00:57:52.597] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:57:52.597+0000] [.info] [src/director/director.go:tracef:794] [00:57:52.597] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:57:52.597+0000] [.info] [src/director/director.go:tracef:794] [00:57:52.597] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:57:52.598+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:57:52.598] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:57:52.753+0000] [.info] [src/director/director.go:tracef:794] [00:57:52.753] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:57:52.753+0000] [.info] [src/director/director.go:tracef:794] [00:57:52.753] because [2021-10-22T00:57:52.753+0000] [.info] [src/director/director.go:tracef:794] [00:57:52.753] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:57:52.753+0000] [.info] [src/director/director.go:tracef:794] [00:57:52.753] because ['currentState.CanBounceStop' = true] [2021-10-22T00:57:52.905+0000] [.info] [src/director/director.go:tracef:794] [00:57:52.905] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:57:52.905+0000] [.info] [src/director/director.go:tracef:794] [00:57:52.905] because [2021-10-22T00:57:52.905+0000] [.info] [src/director/director.go:tracef:794] [00:57:52.905] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:57:52.905+0000] [.info] [src/director/director.go:tracef:794] [00:57:52.905] because ['currentState.Up' = false] [2021-10-22T00:57:52.905+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:57:52.905] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:57:52.905+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:57:52.905] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:57:52.970+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:52.970] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:52.995+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:57:52.995] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:57:53.003+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:57:53.003] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:57:53.003+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:57:53.003] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:57:53.004+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:57:53.004] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:57:53.005+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:57:53.005] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:57:53.005+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:57:53.005] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:57:53.005+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:57:53.005] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:57:53.007+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:57:53.007] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:57:53.220+0000] [.info] [src/director/director.go:tracef:794] [00:57:53.220] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:57:53.220+0000] [.info] [src/director/director.go:tracef:794] [00:57:53.220] because ['currentState.Up' = false] [2021-10-22T00:57:53.405+0000] [.info] [src/director/director.go:tracef:794] [00:57:53.405] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:57:53.405+0000] [.info] [src/director/director.go:tracef:794] [00:57:53.405] because ['currentState.Up' = false] [2021-10-22T00:57:53.405+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:57:53.405] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:57:53.607+0000] [.info] [src/director/director.go:tracef:794] [00:57:53.607] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:57:53.607+0000] [.info] [src/director/director.go:tracef:794] [00:57:53.607] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:57:53.627+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:57:53.627] clusterConfig unchanged [2021-10-22T00:57:53.824+0000] [.info] [src/director/director.go:tracef:794] [00:57:53.824] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:57:53.824+0000] [.info] [src/director/director.go:tracef:794] [00:57:53.824] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:57:53.824+0000] [.info] [src/director/director.go:tracef:794] [00:57:53.824] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:57:53.824+0000] [.info] [src/director/director.go:tracef:794] [00:57:53.824] because ['currentState.Up' = false] [2021-10-22T00:57:53.824+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:57:53.824] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:57:53.824+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:57:53.824] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:57:53.858+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:57:53.858] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:57:53.859+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:57:53.859] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37487 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:57:53.975+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:53.975] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:54.569+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:57:54.569] ...Finished start [pid=68084] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37487,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:57:54.691+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:57:54.691] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 121.753693ms. [2021-10-22T00:57:54.691+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:57:54.691] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37487 (local=false) [2021-10-22T00:57:54.691+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:57:54.691] Dropping existing documents from system.replset [2021-10-22T00:57:54.727+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:57:54.727] clusterConfig unchanged [2021-10-22T00:57:54.815+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:57:54.815] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:57:54.820+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:57:54.820] Inserting oplogTruncateAfterPoint document [2021-10-22T00:57:54.820+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:57:54.820] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:57:54.821+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:57:54.821] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:57:54.821+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:57:54.821] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:57:54.835+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:57:54.835] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:57:54.835+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:57:54.835] Setting the minvalid document to [{_id ObjectID("61720c92523370002becc3ec")} {t -1} {ts {0 1}}] [2021-10-22T00:57:54.847+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:57:54.847] Inserted [{_id ObjectID("61720c92523370002becc3ec")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:57:54.847+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:57:54.847] Inserting new replica set configuration document. [2021-10-22T00:57:54.847+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:57:54.847] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:57:54.859+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:57:54.859] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:57:54.863+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:57:54.862] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:57:54.863+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:57:54.863] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:57:54.863+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:57:54.863] Initiating stopEphemeralMongo [2021-10-22T00:57:54.863+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:57:54.863] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37487 (local=false) [2021-10-22T00:57:54.863+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:57:54.863] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37487 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:57:54.995+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:54.995] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:55.371+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:57:55.371] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.006734ms. [2021-10-22T00:57:55.371+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:57:55.371] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37487 (local=false) [2021-10-22T00:57:55.371+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:57:55.371] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.04µs. [2021-10-22T00:57:55.371+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:57:55.371] Successfully finished running stopEphemeralMongo [2021-10-22T00:57:55.371+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:57:55.371] Successfully stopped existing ephemeral mongod. [2021-10-22T00:57:55.371+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:57:55.371] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:57:55.371+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:57:55.371] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:57:55.404+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:57:55.404] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:57:55.405+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:57:55.405] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55307 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:57:55.856+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:57:55.855] clusterConfig unchanged [2021-10-22T00:57:55.976+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:55.976] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:56.588+0000] [.error] [src/action/start.go:func1:93] [103] [00:57:56.588] Error starting mongod : [00:57:56.588] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55307,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55307 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:57:55.436Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 69661 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:57:55.477+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:57:55.477+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:57:55.477+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55307,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:57:55.479+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:57:55.480+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:57:55.480+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:57:55.552+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:57:55.552+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:57:55.553+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:57:55.605+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:57:55.605+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:57:56.093+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864276:93006][69661:0x7f2f37e3abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:57:56.194+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864276:194081][69661:0x7f2f37e3abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:57:56.294+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864276:294268][69661:0x7f2f37e3abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T00:57:56.408+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864276:408946][69661:0x7f2f37e3abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:57:56.480+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864276:480039][69661:0x7f2f37e3abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:57:56.538+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864276:538445][69661:0x7f2f37e3abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:57:56.538+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864276:538518][69661:0x7f2f37e3abc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:57:56.589+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:57:56.589] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:55307 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:57:56.588] Error starting mongod : [00:57:56.588] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55307,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55307 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:57:55.436Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 69661 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:57:55.477+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:57:55.477+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:57:55.477+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55307,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:57:55.479+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:57:55.480+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:57:55.480+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:57:55.552+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:57:55.552+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:57:55.553+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:57:55.605+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:57:55.605+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:57:56.093+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864276:93006][69661:0x7f2f37e3abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:57:56.194+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:57:56.589+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:57:56.589] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:57:56.589] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:55307 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:57:56.588] Error starting mongod : [00:57:56.588] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55307,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55307 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:57:55.436Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 69661 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:57:55.477+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:57:55.477+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:57:55.477+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55307,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:57:55.479+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:57:55.480+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:57:55.480+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:57:55.552+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:57:55.552+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:57:55.553+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:57:55.605+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:57:55.605+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:57:56.093+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864276:93006][69661:0x7f2f37e3abc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:57:56.589+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:57:56.589] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:57:56.589+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:57:56.589] Initiating stopEphemeralMongo [2021-10-22T00:57:56.589+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:57:56.589] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:57:56.590+0000] [.error] [src/director/director.go:executePlan:962] [00:57:56.590] Failed to apply action. Result = : [00:57:56.589] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:57:56.589] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:55307 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:57:56.588] Error starting mongod : [00:57:56.588] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55307,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55307 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:57:55.436Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 69661 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:57:55.477+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:57:55.477+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:57:55.477+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55307,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:57:55.479+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:57:55.480+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:57:55.480+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:57:55.552+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:57:55.552+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:57:55.553+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:57:55.605+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:57:55.605+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:57:56.093+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:57:56.590+0000] [.error] [src/director/director.go:planAndExecute:567] [00:57:56.590] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:57:56.590] Failed to apply action. Result = : [00:57:56.589] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:57:56.589] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:55307 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:57:56.588] Error starting mongod : [00:57:56.588] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55307,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55307 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:57:55.436Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 69661 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:57:55.477+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:57:55.477+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:57:55.477+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55307,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:57:55.479+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:57:55.480+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:57:55.480+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:57:55.552+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:57:55.552+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:57:55.553+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:57:55.605+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:57:55.605+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:57:56.590+0000] [.error] [src/director/director.go:mainLoop:399] [00:57:56.590] Failed to planAndExecute : [00:57:56.590] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:57:56.590] Failed to apply action. Result = : [00:57:56.589] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:57:56.589] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:55307 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:57:56.588] Error starting mongod : [00:57:56.588] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55307,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55307 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:57:55.436Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 69661 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:57:55.477+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:57:55.477+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:57:55.477+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55307,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:57:55.479+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:57:55.480+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:57:55.480+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:57:55.552+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:57:55.552+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:57:55.553+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:57:55.605+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:57:55.605+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:57:56.976+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:56.976] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:57.076+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:57:57.076] clusterConfig unchanged [2021-10-22T00:57:57.977+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:57.977] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:58.978+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:58.978] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:57:59.977+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:57:59.977] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:00.976+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:00.976] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:01.979+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:01.979] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:02.979+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:02.979] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:03.986+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:03.986] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:04.980+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:04.980] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:05.440+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:58:05.440] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.433447746s [2021-10-22T00:58:05.978+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:05.978] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:06.979+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:06.979] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:07.316+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:58:07.316] clusterConfig unchanged [2021-10-22T00:58:07.979+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:07.979] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:08.980+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:08.980] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:09.980+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:09.980] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:10.981+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:10.981] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:11.981+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:11.981] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:12.982+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:12.982] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:13.983+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:13.983] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:14.984+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:14.984] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:15.991+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:15.991] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:16.986+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:16.986] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:17.458+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:58:17.458] clusterConfig unchanged [2021-10-22T00:58:17.987+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:17.987] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:18.987+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:18.987] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:19.988+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:19.988] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:20.989+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:20.989] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:21.989+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:21.989] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:22.990+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:22.990] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:23.992+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:23.992] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:24.991+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:24.991] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:25.992+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:25.992] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:26.993+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:26.993] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:27.550+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:58:27.550] clusterConfig unchanged [2021-10-22T00:58:27.993+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:27.993] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:28.994+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:28.994] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:29.995+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:29.995] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:30.995+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:30.995] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:31.996+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:31.996] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:32.997+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:32.997] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:33.997+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:33.997] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:34.998+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:34.998] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:36.000+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:36.000] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:37.000+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:37.000] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:37.652+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:58:37.652] clusterConfig unchanged [2021-10-22T00:58:37.999+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:37.999] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:39.006+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:39.006] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:40.001+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:40.001] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:41.001+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:41.001] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:41.287+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:58:41.287] memory:rss=50761728 alloc=8137672 totalalloc=3296909944 mallocs=42451828 frees=42395522 heapinuse=14041088 heapobjects=56306 gcpausetotalns=1424706166 gcpauselastns=133703 gcnum=528 utimens=58033261000 stimens=18291693000 minflt=160553 majflt=405 [2021-10-22T00:58:42.002+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:42.002] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:43.002+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:43.002] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:44.003+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:44.003] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:45.025+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:45.025] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:46.008+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:46.008] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:47.008+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:47.008] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:47.764+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:58:47.764] clusterConfig unchanged [2021-10-22T00:58:48.014+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:48.014] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:49.008+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:49.008] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:50.009+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:50.009] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:51.010+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:51.010] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:52.011+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:52.011] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:53.011+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:53.011] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:54.011+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:54.011] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:55.012+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:55.012] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:56.029+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:56.029] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:56.752+0000] [.info] [src/director/director.go:computePlan:280] [00:58:56.752] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T00:58:56.752+0000] [.info] [src/director/director.go:tracef:794] [00:58:56.752] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:58:56.752+0000] [.info] [src/director/director.go:tracef:794] [00:58:56.752] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T00:58:56.752+0000] [.info] [src/director/director.go:tracef:794] [00:58:56.752] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:58:56.752+0000] [.info] [src/director/director.go:tracef:794] [00:58:56.752] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T00:58:56.752+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:58:56.752] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:58:56.930+0000] [.info] [src/director/director.go:tracef:794] [00:58:56.930] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:58:56.930+0000] [.info] [src/director/director.go:tracef:794] [00:58:56.930] because [2021-10-22T00:58:56.931+0000] [.info] [src/director/director.go:tracef:794] [00:58:56.931] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:58:56.931+0000] [.info] [src/director/director.go:tracef:794] [00:58:56.931] because ['currentState.CanBounceStop' = true] [2021-10-22T00:58:57.015+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:57.015] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:57.089+0000] [.info] [src/director/director.go:tracef:794] [00:58:57.089] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:58:57.089+0000] [.info] [src/director/director.go:tracef:794] [00:58:57.089] because [2021-10-22T00:58:57.089+0000] [.info] [src/director/director.go:tracef:794] [00:58:57.089] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:58:57.089+0000] [.info] [src/director/director.go:tracef:794] [00:58:57.089] because ['currentState.Up' = false] [2021-10-22T00:58:57.089+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [00:58:57.089] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:58:57.089+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [00:58:57.089] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T00:58:57.168+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:58:57.168] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T00:58:57.168+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [00:58:57.168] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T00:58:57.176+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:58:57.176] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T00:58:57.177+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:58:57.177] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T00:58:57.178+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [00:58:57.178] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:58:57.178+0000] [.info] [src/util/io.go:PreWarmDisk:29] [00:58:57.178] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:58:57.179+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:58:57.179] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:58:57.180+0000] [.info] [src/util/io.go:PreWarmDisk:45] [00:58:57.180] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T00:58:57.389+0000] [.info] [src/director/director.go:tracef:794] [00:58:57.389] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:58:57.389+0000] [.info] [src/director/director.go:tracef:794] [00:58:57.389] because ['currentState.Up' = false] [2021-10-22T00:58:57.606+0000] [.info] [src/director/director.go:tracef:794] [00:58:57.606] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:58:57.606+0000] [.info] [src/director/director.go:tracef:794] [00:58:57.606] because ['currentState.Up' = false] [2021-10-22T00:58:57.606+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [00:58:57.606] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T00:58:57.814+0000] [.info] [src/director/director.go:tracef:794] [00:58:57.814] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T00:58:57.814+0000] [.info] [src/director/director.go:tracef:794] [00:58:57.814] because ['desiredState.ProcessType' = mongod] [2021-10-22T00:58:57.965+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:58:57.965] clusterConfig unchanged [2021-10-22T00:58:58.020+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:58.020] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:58.033+0000] [.info] [src/director/director.go:tracef:794] [00:58:58.033] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:58:58.033+0000] [.info] [src/director/director.go:tracef:794] [00:58:58.033] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T00:58:58.034+0000] [.info] [src/director/director.go:tracef:794] [00:58:58.034] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T00:58:58.034+0000] [.info] [src/director/director.go:tracef:794] [00:58:58.034] because ['currentState.Up' = false] [2021-10-22T00:58:58.034+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [00:58:58.034] Initiating startEphemeralMongoAsStandalone [2021-10-22T00:58:58.034+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [00:58:58.034] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:58:58.070+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:58:58.070] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:58:58.070+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:58:58.070] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49757 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:58:58.796+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [00:58:58.796] ...Finished start [pid=14935] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49757,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T00:58:58.919+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:58:58.919] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 122.352737ms. [2021-10-22T00:58:58.919+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [00:58:58.919] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49757 (local=false) [2021-10-22T00:58:58.919+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [00:58:58.919] Dropping existing documents from system.replset [2021-10-22T00:58:59.019+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:58:59.019] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:58:59.048+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [00:58:59.048] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T00:58:59.052+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [00:58:59.052] Inserting oplogTruncateAfterPoint document [2021-10-22T00:58:59.052+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [00:58:59.052] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T00:58:59.053+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [00:58:59.053] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T00:58:59.053+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [00:58:59.053] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T00:58:59.065+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:58:59.065] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T00:58:59.065+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [00:58:59.065] Setting the minvalid document to [{_id ObjectID("61720cd3523370002becc3fa")} {t -1} {ts {0 1}}] [2021-10-22T00:58:59.078+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:58:59.078] Inserted [{_id ObjectID("61720cd3523370002becc3fa")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T00:58:59.078+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [00:58:59.078] Inserting new replica set configuration document. [2021-10-22T00:58:59.078+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [00:58:59.078] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T00:58:59.088+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:58:59.088] clusterConfig unchanged [2021-10-22T00:58:59.092+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [00:58:59.092] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T00:58:59.095+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [00:58:59.095] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T00:58:59.095+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [00:58:59.095] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T00:58:59.095+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:58:59.095] Initiating stopEphemeralMongo [2021-10-22T00:58:59.095+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [00:58:59.095] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49757 (local=false) [2021-10-22T00:58:59.095+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [00:58:59.095] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49757 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T00:58:59.452+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:58:59.452] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.923854ms. [2021-10-22T00:58:59.452+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [00:58:59.452] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49757 (local=false) [2021-10-22T00:58:59.452+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [00:58:59.452] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.468µs. [2021-10-22T00:58:59.453+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [00:58:59.453] Successfully finished running stopEphemeralMongo [2021-10-22T00:58:59.453+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [00:58:59.453] Successfully stopped existing ephemeral mongod. [2021-10-22T00:58:59.453+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [00:58:59.453] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T00:58:59.453+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [00:58:59.453] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T00:58:59.486+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [00:58:59.486] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T00:58:59.487+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [00:58:59.487] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51839 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T00:59:00.021+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:00.020] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:00.259+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:59:00.259] clusterConfig unchanged [2021-10-22T00:59:00.669+0000] [.error] [src/action/start.go:func1:93] [103] [00:59:00.669] Error starting mongod : [00:59:00.669] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51839,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51839 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:58:59.517Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 16460 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:58:59.537+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:58:59.537+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:58:59.537+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51839,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:58:59.539+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:58:59.541+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:58:59.543+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:58:59.587+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:58:59.587+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:58:59.587+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:58:59.633+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:58:59.633+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:59:00.161+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864340:161682][16460:0x7fb8ed41bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:59:00.269+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864340:269098][16460:0x7fb8ed41bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:59:00.370+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864340:370572][16460:0x7fb8ed41bbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/33920 to 2/256"}} {"t":{"$date":"2021-10-22T00:59:00.482+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864340:482390][16460:0x7fb8ed41bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:59:00.559+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864340:559154][16460:0x7fb8ed41bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T00:59:00.618+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864340:618490][16460:0x7fb8ed41bbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T00:59:00.618+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864340:618568][16460:0x7fb8ed41bbc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T00:59:00.669+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [00:59:00.669] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51839 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:59:00.669] Error starting mongod : [00:59:00.669] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51839,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51839 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:58:59.517Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 16460 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:58:59.537+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:58:59.537+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:58:59.537+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51839,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:58:59.539+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:58:59.541+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:58:59.543+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:58:59.587+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:58:59.587+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:58:59.587+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:58:59.633+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:58:59.633+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:59:00.161+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864340:161682][16460:0x7fb8ed41bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T00:59:00.269+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T00:59:00.670+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [00:59:00.669] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:59:00.669] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51839 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:59:00.669] Error starting mongod : [00:59:00.669] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51839,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51839 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:58:59.517Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 16460 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:58:59.537+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:58:59.537+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:58:59.537+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51839,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:58:59.539+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:58:59.541+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:58:59.543+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:58:59.587+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:58:59.587+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:58:59.587+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:58:59.633+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:58:59.633+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:59:00.161+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864340:161682][16460:0x7fb8ed41bbc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:59:00.670+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [00:59:00.670] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T00:59:00.670+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [00:59:00.670] Initiating stopEphemeralMongo [2021-10-22T00:59:00.670+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [00:59:00.670] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T00:59:00.670+0000] [.error] [src/director/director.go:executePlan:962] [00:59:00.670] Failed to apply action. Result = : [00:59:00.669] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:59:00.669] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51839 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:59:00.669] Error starting mongod : [00:59:00.669] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51839,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51839 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:58:59.517Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 16460 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:58:59.537+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:58:59.537+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:58:59.537+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51839,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:58:59.539+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:58:59.541+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:58:59.543+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:58:59.587+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:58:59.587+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:58:59.587+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:58:59.633+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:58:59.633+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:59:00.161+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:59:00.670+0000] [.error] [src/director/director.go:planAndExecute:567] [00:59:00.670] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:59:00.670] Failed to apply action. Result = : [00:59:00.669] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:59:00.669] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51839 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:59:00.669] Error starting mongod : [00:59:00.669] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51839,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51839 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:58:59.517Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 16460 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:58:59.537+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:58:59.537+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:58:59.537+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51839,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:58:59.539+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:58:59.541+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:58:59.543+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:58:59.587+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:58:59.587+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:58:59.587+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:58:59.633+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:58:59.633+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:59:00.670+0000] [.error] [src/director/director.go:mainLoop:399] [00:59:00.670] Failed to planAndExecute : [00:59:00.670] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [00:59:00.670] Failed to apply action. Result = : [00:59:00.669] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [00:59:00.669] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51839 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [00:59:00.669] Error starting mongod : [00:59:00.669] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51839,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51839 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T00:58:59.517Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 16460 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T00:58:59.537+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T00:58:59.537+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T00:58:59.537+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51839,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T00:58:59.539+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:58:59.541+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T00:58:59.543+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T00:58:59.587+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:58:59.587+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T00:58:59.587+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T00:58:59.633+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T00:58:59.633+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T00:59:01.021+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:01.021] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:01.519+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:59:01.519] clusterConfig unchanged [2021-10-22T00:59:02.034+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:02.034] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:03.022+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:03.022] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:04.023+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:04.023] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:05.023+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:05.023] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:06.021+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:06.021] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:07.023+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:07.023] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:08.024+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:08.024] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:09.027+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:09.027] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:09.494+0000] [.info] [src/util/io.go:PreWarmDisk:56] [00:59:09.494] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.313963908s [2021-10-22T00:59:10.023+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:10.023] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:11.024+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:11.024] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:11.661+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:59:11.661] clusterConfig unchanged [2021-10-22T00:59:12.025+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:12.025] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:13.025+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:13.025] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:14.025+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:14.025] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:15.026+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:15.026] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:16.030+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:16.030] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:17.029+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:17.029] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:18.027+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:18.027] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:19.028+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:19.028] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:20.029+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:20.029] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:21.034+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:21.034] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:21.753+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:59:21.753] clusterConfig unchanged [2021-10-22T00:59:22.030+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:22.030] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:23.030+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:23.030] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:24.031+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:24.031] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:25.031+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:25.031] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:26.032+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:26.032] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:27.032+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:27.032] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:28.033+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:28.033] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:29.033+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:29.033] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:30.035+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:30.035] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:31.035+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:31.035] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:31.865+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:59:31.865] clusterConfig unchanged [2021-10-22T00:59:32.035+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:32.035] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:33.042+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:33.042] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:34.036+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:34.036] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:35.037+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:35.037] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:36.037+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:36.037] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:37.037+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:37.037] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:38.038+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:38.038] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:39.038+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:39.038] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:40.039+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:40.039] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:41.039+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:41.039] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:41.288+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [00:59:41.288] memory:rss=51658752 alloc=11242896 totalalloc=3382840032 mallocs=43537657 frees=43448555 heapinuse=15122432 heapobjects=89102 gcpausetotalns=1426322105 gcpauselastns=136008 gcnum=541 utimens=59693433000 stimens=18869906000 minflt=164557 majflt=405 [2021-10-22T00:59:42.001+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:59:42.001] clusterConfig unchanged [2021-10-22T00:59:42.042+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:42.042] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:43.040+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:43.040] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:44.040+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:44.040] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:44.181+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [00:59:44.181] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51839 (local=false) because it is not up [2021-10-22T00:59:45.041+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:45.041] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:46.041+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:46.041] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:47.042+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:47.042] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:48.043+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:48.043] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:49.043+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:49.043] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:50.044+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:50.044] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:51.043+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:51.043] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:52.049+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:52.049] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:52.129+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [00:59:52.129] clusterConfig unchanged [2021-10-22T00:59:53.044+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:53.044] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:54.048+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:54.048] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:55.046+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:55.046] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:56.051+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:56.051] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:57.047+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:57.047] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:58.047+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:58.047] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T00:59:59.048+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [00:59:59.048] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:00.048+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:00.048] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:00.841+0000] [.info] [src/director/director.go:computePlan:280] [01:00:00.841] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:00:00.841+0000] [.info] [src/director/director.go:tracef:794] [01:00:00.841] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:00:00.841+0000] [.info] [src/director/director.go:tracef:794] [01:00:00.841] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:00:00.841+0000] [.info] [src/director/director.go:tracef:794] [01:00:00.841] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:00:00.841+0000] [.info] [src/director/director.go:tracef:794] [01:00:00.841] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:00:00.841+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:00:00.841] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:00:01.004+0000] [.info] [src/director/director.go:tracef:794] [01:00:01.004] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:00:01.004+0000] [.info] [src/director/director.go:tracef:794] [01:00:01.004] because [2021-10-22T01:00:01.004+0000] [.info] [src/director/director.go:tracef:794] [01:00:01.004] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:00:01.004+0000] [.info] [src/director/director.go:tracef:794] [01:00:01.004] because ['currentState.CanBounceStop' = true] [2021-10-22T01:00:01.066+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:01.066] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:01.173+0000] [.info] [src/director/director.go:tracef:794] [01:00:01.173] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:00:01.173+0000] [.info] [src/director/director.go:tracef:794] [01:00:01.173] because [2021-10-22T01:00:01.173+0000] [.info] [src/director/director.go:tracef:794] [01:00:01.173] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:00:01.173+0000] [.info] [src/director/director.go:tracef:794] [01:00:01.173] because ['currentState.Up' = false] [2021-10-22T01:00:01.173+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:00:01.173] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:00:01.173+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:00:01.173] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:00:01.256+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:00:01.256] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:00:01.264+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:00:01.264] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:00:01.265+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:00:01.265] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:00:01.266+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:00:01.266] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:00:01.267+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:00:01.267] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:00:01.267+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:00:01.267] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:00:01.267+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:00:01.267] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:00:01.268+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:00:01.268] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:00:01.525+0000] [.info] [src/director/director.go:tracef:794] [01:00:01.525] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:00:01.525+0000] [.info] [src/director/director.go:tracef:794] [01:00:01.525] because ['currentState.Up' = false] [2021-10-22T01:00:01.979+0000] [.info] [src/director/director.go:tracef:794] [01:00:01.979] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:00:01.979+0000] [.info] [src/director/director.go:tracef:794] [01:00:01.979] because ['currentState.Up' = false] [2021-10-22T01:00:01.979+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:00:01.979] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:00:02.054+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:02.054] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:02.285+0000] [.info] [src/director/director.go:tracef:794] [01:00:02.285] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:00:02.285+0000] [.info] [src/director/director.go:tracef:794] [01:00:02.285] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:00:02.319+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:00:02.319] clusterConfig unchanged [2021-10-22T01:00:02.508+0000] [.info] [src/director/director.go:tracef:794] [01:00:02.508] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:00:02.508+0000] [.info] [src/director/director.go:tracef:794] [01:00:02.508] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:00:02.508+0000] [.info] [src/director/director.go:tracef:794] [01:00:02.508] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:00:02.508+0000] [.info] [src/director/director.go:tracef:794] [01:00:02.508] because ['currentState.Up' = false] [2021-10-22T01:00:02.508+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:00:02.508] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:00:02.508+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:00:02.508] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:00:02.546+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:00:02.546] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:00:02.546+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:00:02.546] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34115 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:00:03.055+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:03.055] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:03.268+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:00:03.268] ...Finished start [pid=36285] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34115,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:00:03.391+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:00:03.390] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 122.608387ms. [2021-10-22T01:00:03.391+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:00:03.391] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34115 (local=false) [2021-10-22T01:00:03.391+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:00:03.391] Dropping existing documents from system.replset [2021-10-22T01:00:03.432+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:00:03.432] clusterConfig unchanged [2021-10-22T01:00:03.510+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:00:03.510] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:00:03.513+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:00:03.513] Inserting oplogTruncateAfterPoint document [2021-10-22T01:00:03.513+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:00:03.513] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:00:03.514+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:00:03.514] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:00:03.514+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:00:03.514] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:00:03.528+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:00:03.528] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:00:03.528+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:00:03.528] Setting the minvalid document to [{_id ObjectID("61720d13523370002becc408")} {t -1} {ts {0 1}}] [2021-10-22T01:00:03.542+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:00:03.542] Inserted [{_id ObjectID("61720d13523370002becc408")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:00:03.542+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:00:03.542] Inserting new replica set configuration document. [2021-10-22T01:00:03.542+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:00:03.542] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:00:03.555+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:00:03.555] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:00:03.559+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:00:03.559] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:00:03.559+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:00:03.559] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:00:03.559+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:00:03.559] Initiating stopEphemeralMongo [2021-10-22T01:00:03.559+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:00:03.559] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34115 (local=false) [2021-10-22T01:00:03.559+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:00:03.559] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34115 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:00:03.944+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:00:03.944] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.917299ms. [2021-10-22T01:00:03.944+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:00:03.944] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34115 (local=false) [2021-10-22T01:00:03.944+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:00:03.944] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 31.528µs. [2021-10-22T01:00:03.944+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:00:03.944] Successfully finished running stopEphemeralMongo [2021-10-22T01:00:03.945+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:00:03.944] Successfully stopped existing ephemeral mongod. [2021-10-22T01:00:03.945+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:00:03.945] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:00:03.945+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:00:03.945] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:00:03.982+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:00:03.982] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:00:03.983+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:00:03.982] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32928 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:00:04.059+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:04.059] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:04.545+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:00:04.545] clusterConfig unchanged [2021-10-22T01:00:05.061+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:05.061] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:05.200+0000] [.error] [src/action/start.go:func1:93] [103] [01:00:05.200] Error starting mongod : [01:00:05.200] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32928,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32928 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 37722 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:00:04.059+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:00:04.059+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:00:04.060+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32928,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:00:04.061+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:00:04.063+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:00:04.085+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:00:04.100+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:00:04.100+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:00:04.100+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:00:04.164+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:00:04.164+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:00:04.678+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864404:678313][37722:0x7f2905699bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:00:04.786+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864404:786349][37722:0x7f2905699bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:00:04.888+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864404:888831][37722:0x7f2905699bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/34048 to 2/256"}} {"t":{"$date":"2021-10-22T01:00:05.004+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864405:4187][37722:0x7f2905699bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:00:05.079+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864405:79096][37722:0x7f2905699bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:00:05.149+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864405:149828][37722:0x7f2905699bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:00:05.149+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864405:149900][37722:0x7f2905699bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:00:05.151+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:00:05.201+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:00:05.201] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:32928 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:00:05.200] Error starting mongod : [01:00:05.200] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32928,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32928 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 37722 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:00:04.059+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:00:04.059+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:00:04.060+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32928,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:00:04.061+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:00:04.063+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:00:04.085+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:00:04.100+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:00:04.100+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:00:04.100+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:00:04.164+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:00:04.164+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:00:04.678+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864404:678313][37722:0x7f2905699bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:00:04.786+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864404:786349][37722:0x7f2905699bc0], txn-recover: ...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:00:05.201+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:00:05.201] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:00:05.201] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:32928 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:00:05.200] Error starting mongod : [01:00:05.200] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32928,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32928 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 37722 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:00:04.059+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:00:04.059+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:00:04.060+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32928,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:00:04.061+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:00:04.063+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:00:04.085+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:00:04.100+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:00:04.100+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:00:04.100+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:00:04.164+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:00:04.164+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:00:04.678+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864404:678313][37722:0x7f2905699bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:00:04.786+00:00"},"s":"I", "c":"STORAGE", "id":22430, "... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:00:05.201+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:00:05.201] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:00:05.201+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:00:05.201] Initiating stopEphemeralMongo [2021-10-22T01:00:05.201+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:00:05.201] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:00:05.201+0000] [.error] [src/director/director.go:executePlan:962] [01:00:05.201] Failed to apply action. Result = : [01:00:05.201] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:00:05.201] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:32928 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:00:05.200] Error starting mongod : [01:00:05.200] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32928,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32928 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 37722 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:00:04.059+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:00:04.059+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:00:04.060+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32928,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:00:04.061+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:00:04.063+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:00:04.085+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:00:04.100+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:00:04.100+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:00:04.100+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:00:04.164+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:00:04.164+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:00:04.678+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864404:678313][37722:0x7f2905699bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:00:05.201+0000] [.error] [src/director/director.go:planAndExecute:567] [01:00:05.201] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:00:05.201] Failed to apply action. Result = : [01:00:05.201] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:00:05.201] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:32928 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:00:05.200] Error starting mongod : [01:00:05.200] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32928,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32928 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 37722 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:00:04.059+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:00:04.059+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:00:04.060+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32928,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:00:04.061+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:00:04.063+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:00:04.085+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:00:04.100+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:00:04.100+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:00:04.100+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:00:04.164+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:00:04.164+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:00:04.678+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger messa... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:00:05.202+0000] [.error] [src/director/director.go:mainLoop:399] [01:00:05.202] Failed to planAndExecute : [01:00:05.201] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:00:05.201] Failed to apply action. Result = : [01:00:05.201] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:00:05.201] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:32928 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:00:05.200] Error starting mongod : [01:00:05.200] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32928,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32928 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 37722 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:00:04.059+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:00:04.059+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:00:04.060+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32928,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:00:04.061+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:00:04.063+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:00:04.085+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:00:04.100+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:00:04.100+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:00:04.100+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:00:04.164+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:00:04.164+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:00:04.678+00:00"},"s":"I", "c... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:00:05.805+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:00:05.805] clusterConfig unchanged [2021-10-22T01:00:06.060+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:06.060] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:07.058+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:07.058] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:08.060+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:08.060] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:09.061+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:09.061] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:10.061+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:10.061] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:11.061+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:11.061] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:12.063+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:12.063] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:13.065+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:13.065] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:13.744+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:00:13.744] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.475511953s [2021-10-22T01:00:14.064+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:14.064] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:15.065+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:15.065] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:15.937+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:00:15.937] clusterConfig unchanged [2021-10-22T01:00:16.063+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:16.063] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:17.064+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:17.064] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:18.065+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:18.064] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:19.065+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:19.065] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:20.066+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:20.066] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:21.066+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:21.066] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:22.067+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:22.067] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:23.068+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:23.068] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:24.071+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:24.071] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:25.071+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:25.071] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:26.049+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:00:26.049] clusterConfig unchanged [2021-10-22T01:00:26.069+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:26.069] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:27.076+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:27.076] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:28.072+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:28.072] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:29.071+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:29.071] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:30.076+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:30.076] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:31.073+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:31.073] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:32.075+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:32.075] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:33.075+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:33.075] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:34.074+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:34.074] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:35.076+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:35.076] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:36.076+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:36.076] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:36.181+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:00:36.181] clusterConfig unchanged [2021-10-22T01:00:37.077+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:37.077] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:38.078+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:38.078] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:39.083+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:39.083] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:40.078+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:40.078] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:41.079+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:41.079] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:41.289+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:00:41.289] memory:rss=51703808 alloc=8200392 totalalloc=3467577248 mallocs=44611787 frees=44555213 heapinuse=14319616 heapobjects=56574 gcpausetotalns=1428224025 gcpauselastns=143520 gcnum=554 utimens=61298609000 stimens=19570621000 minflt=168699 majflt=405 [2021-10-22T01:00:42.079+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:42.079] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:43.079+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:43.079] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:44.081+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:44.081] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:45.081+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:45.081] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:46.082+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:46.082] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:46.273+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:00:46.273] clusterConfig unchanged [2021-10-22T01:00:47.082+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:47.082] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:48.083+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:48.083] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:49.083+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:49.083] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:50.094+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:50.094] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:51.085+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:51.085] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:52.085+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:52.085] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:53.085+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:53.085] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:54.086+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:54.086] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:55.087+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:55.087] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:56.088+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:56.088] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:56.385+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:00:56.385] clusterConfig unchanged [2021-10-22T01:00:57.088+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:57.088] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:58.089+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:58.088] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:00:59.089+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:00:59.089] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:00.089+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:00.089] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:01.090+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:01.090] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:02.092+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:02.092] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:03.091+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:03.091] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:04.091+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:04.091] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:05.092+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:05.092] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:05.385+0000] [.info] [src/director/director.go:computePlan:280] [01:01:05.385] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:01:05.385+0000] [.info] [src/director/director.go:tracef:794] [01:01:05.385] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:01:05.385+0000] [.info] [src/director/director.go:tracef:794] [01:01:05.385] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:01:05.385+0000] [.info] [src/director/director.go:tracef:794] [01:01:05.385] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:01:05.385+0000] [.info] [src/director/director.go:tracef:794] [01:01:05.385] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:01:05.385+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:01:05.385] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:01:05.551+0000] [.info] [src/director/director.go:tracef:794] [01:01:05.551] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:01:05.551+0000] [.info] [src/director/director.go:tracef:794] [01:01:05.551] because [2021-10-22T01:01:05.551+0000] [.info] [src/director/director.go:tracef:794] [01:01:05.551] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:01:05.551+0000] [.info] [src/director/director.go:tracef:794] [01:01:05.551] because ['currentState.CanBounceStop' = true] [2021-10-22T01:01:05.709+0000] [.info] [src/director/director.go:tracef:794] [01:01:05.709] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:01:05.709+0000] [.info] [src/director/director.go:tracef:794] [01:01:05.709] because [2021-10-22T01:01:05.709+0000] [.info] [src/director/director.go:tracef:794] [01:01:05.709] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:01:05.709+0000] [.info] [src/director/director.go:tracef:794] [01:01:05.709] because ['currentState.Up' = false] [2021-10-22T01:01:05.710+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:01:05.710] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:01:05.710+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:01:05.710] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:01:05.896+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:01:05.896] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:01:05.904+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:01:05.904] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:01:05.905+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:01:05.905] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:01:05.906+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:01:05.906] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:01:05.906+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:01:05.906] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:01:05.906+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:01:05.906] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:01:05.908+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:01:05.908] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:01:06.093+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:06.093] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:06.144+0000] [.info] [src/director/director.go:tracef:794] [01:01:06.144] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:01:06.144+0000] [.info] [src/director/director.go:tracef:794] [01:01:06.144] because ['currentState.Up' = false] [2021-10-22T01:01:06.353+0000] [.info] [src/director/director.go:tracef:794] [01:01:06.353] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:01:06.353+0000] [.info] [src/director/director.go:tracef:794] [01:01:06.353] because ['currentState.Up' = false] [2021-10-22T01:01:06.353+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:01:06.353] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:01:06.547+0000] [.info] [src/director/director.go:tracef:794] [01:01:06.547] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:01:06.547+0000] [.info] [src/director/director.go:tracef:794] [01:01:06.547] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:01:06.595+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:01:06.595] clusterConfig unchanged [2021-10-22T01:01:06.769+0000] [.info] [src/director/director.go:tracef:794] [01:01:06.769] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:01:06.769+0000] [.info] [src/director/director.go:tracef:794] [01:01:06.769] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:01:06.769+0000] [.info] [src/director/director.go:tracef:794] [01:01:06.769] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:01:06.769+0000] [.info] [src/director/director.go:tracef:794] [01:01:06.769] because ['currentState.Up' = false] [2021-10-22T01:01:06.770+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:01:06.770] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:01:06.770+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:01:06.770] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:01:06.802+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:01:06.802] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:01:06.803+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:01:06.803] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45441 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:01:07.096+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:07.096] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:07.526+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:01:07.526] ...Finished start [pid=56463] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45441,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:01:07.685+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:01:07.685] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 158.75561ms. [2021-10-22T01:01:07.685+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:01:07.685] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45441 (local=false) [2021-10-22T01:01:07.685+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:01:07.685] Dropping existing documents from system.replset [2021-10-22T01:01:07.747+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:01:07.747] clusterConfig unchanged [2021-10-22T01:01:07.880+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:01:07.880] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:01:07.885+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:01:07.885] Inserting oplogTruncateAfterPoint document [2021-10-22T01:01:07.885+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:01:07.885] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:01:07.885+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:01:07.885] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:01:07.886+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:01:07.885] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:01:07.913+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:01:07.913] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:01:07.913+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:01:07.913] Setting the minvalid document to [{_id ObjectID("61720d53523370002becc416")} {t -1} {ts {0 1}}] [2021-10-22T01:01:07.926+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:01:07.926] Inserted [{_id ObjectID("61720d53523370002becc416")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:01:07.926+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:01:07.926] Inserting new replica set configuration document. [2021-10-22T01:01:07.926+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:01:07.926] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:01:07.938+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:01:07.938] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:01:07.943+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:01:07.943] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:01:07.943+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:01:07.943] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:01:07.943+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:01:07.943] Initiating stopEphemeralMongo [2021-10-22T01:01:07.943+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:01:07.943] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45441 (local=false) [2021-10-22T01:01:07.943+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:01:07.943] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45441 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:01:08.099+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:08.099] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:08.290+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:01:08.290] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.077166ms. [2021-10-22T01:01:08.290+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:01:08.290] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45441 (local=false) [2021-10-22T01:01:08.290+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:01:08.290] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.951µs. [2021-10-22T01:01:08.290+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:01:08.290] Successfully finished running stopEphemeralMongo [2021-10-22T01:01:08.290+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:01:08.290] Successfully stopped existing ephemeral mongod. [2021-10-22T01:01:08.290+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:01:08.290] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:01:08.290+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:01:08.290] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:01:08.324+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:01:08.324] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:01:08.325+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:01:08.325] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35196 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:01:08.880+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:01:08.880] clusterConfig unchanged [2021-10-22T01:01:09.098+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:09.098] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:09.512+0000] [.error] [src/action/start.go:func1:93] [103] [01:01:09.512] Error starting mongod : [01:01:09.512] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35196,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35196 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 57997 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:01:08.379+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:01:08.379+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:01:08.379+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35196,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:01:08.381+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:01:08.383+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:01:08.393+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:01:08.406+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:01:08.406+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:01:08.406+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:01:08.455+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:01:08.455+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:01:08.998+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864468:998375][57997:0x7fc75c5bfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:01:09.104+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864469:104823][57997:0x7fc75c5bfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:01:09.209+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864469:209659][57997:0x7fc75c5bfbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T01:01:09.323+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864469:323329][57997:0x7fc75c5bfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:01:09.400+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864469:400029][57997:0x7fc75c5bfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:01:09.461+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864469:461518][57997:0x7fc75c5bfbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:01:09.461+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864469:461592][57997:0x7fc75c5bfbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:01:09.463+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:01:09.512+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:01:09.512] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35196 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:01:09.512] Error starting mongod : [01:01:09.512] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35196,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35196 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 57997 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:01:08.379+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:01:08.379+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:01:08.379+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35196,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:01:08.381+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:01:08.383+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:01:08.393+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:01:08.406+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:01:08.406+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:01:08.406+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:01:08.455+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:01:08.455+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:01:08.998+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864468:998375][57997:0x7fc75c5bfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:01:09.104+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864469:104823][57997:0x7fc75c5bfbc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:01:09.512+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:01:09.512] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:01:09.512] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35196 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:01:09.512] Error starting mongod : [01:01:09.512] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35196,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35196 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 57997 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:01:08.379+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:01:08.379+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:01:08.379+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35196,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:01:08.381+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:01:08.383+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:01:08.393+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:01:08.406+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:01:08.406+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:01:08.406+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:01:08.455+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:01:08.455+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:01:08.998+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864468:998375][57997:0x7fc75c5bfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:01:09.104+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:01:09.512+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:01:09.512] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:01:09.513+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:01:09.512] Initiating stopEphemeralMongo [2021-10-22T01:01:09.513+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:01:09.513] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:01:09.513+0000] [.error] [src/director/director.go:executePlan:962] [01:01:09.513] Failed to apply action. Result = : [01:01:09.512] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:01:09.512] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35196 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:01:09.512] Error starting mongod : [01:01:09.512] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35196,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35196 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 57997 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:01:08.379+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:01:08.379+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:01:08.379+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35196,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:01:08.381+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:01:08.383+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:01:08.393+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:01:08.406+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:01:08.406+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:01:08.406+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:01:08.455+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:01:08.455+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:01:08.998+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864468:998375][57997:0x7fc75c5bfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:01:09.513+0000] [.error] [src/director/director.go:planAndExecute:567] [01:01:09.513] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:01:09.513] Failed to apply action. Result = : [01:01:09.512] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:01:09.512] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35196 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:01:09.512] Error starting mongod : [01:01:09.512] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35196,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35196 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 57997 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:01:08.379+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:01:08.379+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:01:08.379+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35196,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:01:08.381+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:01:08.383+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:01:08.393+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:01:08.406+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:01:08.406+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:01:08.406+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:01:08.455+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:01:08.455+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:01:08.998+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:01:09.513+0000] [.error] [src/director/director.go:mainLoop:399] [01:01:09.513] Failed to planAndExecute : [01:01:09.513] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:01:09.513] Failed to apply action. Result = : [01:01:09.512] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:01:09.512] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35196 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:01:09.512] Error starting mongod : [01:01:09.512] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35196,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35196 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 57997 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:01:08.379+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:01:08.379+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:01:08.379+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35196,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:01:08.381+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:01:08.383+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:01:08.393+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:01:08.406+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:01:08.406+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:01:08.406+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:01:08.455+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:01:08.455+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:01:08.998+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:01:10.098+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:10.098] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:10.179+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:01:10.179] clusterConfig unchanged [2021-10-22T01:01:11.111+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:11.111] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:12.111+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:12.111] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:13.111+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:13.111] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:14.112+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:14.112] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:15.113+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:15.113] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:16.110+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:16.110] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:17.113+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:17.113] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:18.113+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:18.113] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:18.301+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:01:18.301] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.393013968s [2021-10-22T01:01:18.557+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:01:18.557] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:01:18.559+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:18.559] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:01:18.559+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [01:01:18.559] Could not get version : [01:01:18.559] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:01:18.560+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [01:01:18.559] Error getting version for log lines. Skipping : [01:01:18.559] Could not get version : [01:01:18.559] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:01:19.118+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:19.118] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:20.112+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:20.112] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:20.285+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:01:20.285] clusterConfig unchanged [2021-10-22T01:01:21.112+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:21.112] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:22.113+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:22.113] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:23.113+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:23.113] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:24.114+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:24.114] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:25.114+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:25.114] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:26.114+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:26.114] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:27.115+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:27.115] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:28.115+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:28.115] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:29.116+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:29.116] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:30.117+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:30.117] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:30.383+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:01:30.383] clusterConfig unchanged [2021-10-22T01:01:31.122+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:31.122] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:32.118+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:32.118] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:33.118+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:33.118] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:34.119+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:34.119] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:35.123+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:35.123] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:36.120+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:36.120] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:37.120+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:37.120] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:38.121+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:38.121] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:39.121+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:39.121] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:40.122+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:40.122] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:40.505+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:01:40.505] clusterConfig unchanged [2021-10-22T01:01:41.122+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:41.122] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:41.290+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:01:41.290] memory:rss=51138560 alloc=12193904 totalalloc=3554739440 mallocs=45699187 frees=45599880 heapinuse=15687680 heapobjects=99307 gcpausetotalns=1430252861 gcpauselastns=48420 gcnum=567 utimens=63006240000 stimens=20221492000 minflt=173109 majflt=405 [2021-10-22T01:01:42.125+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:42.125] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:43.124+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:43.124] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:44.125+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:44.125] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:45.125+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:45.125] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:46.127+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:46.127] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:47.127+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:47.127] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:48.128+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:48.128] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:49.129+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:49.129] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:50.129+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:50.129] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:50.637+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:01:50.637] clusterConfig unchanged [2021-10-22T01:01:51.130+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:51.130] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:52.131+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:52.131] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:53.134+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:53.134] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:54.132+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:54.132] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:55.133+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:55.133] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:56.134+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:56.134] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:57.134+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:57.134] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:58.134+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:58.134] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:01:59.134+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:01:59.134] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:00.134+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:00.134] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:00.759+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:02:00.759] clusterConfig unchanged [2021-10-22T01:02:01.140+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:01.140] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:02.136+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:02.136] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:03.137+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:03.137] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:04.137+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:04.137] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:05.138+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:05.138] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:06.138+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:06.138] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:07.152+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:07.152] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:08.139+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:08.139] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:09.140+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:09.140] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:09.688+0000] [.info] [src/director/director.go:computePlan:280] [01:02:09.688] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:02:09.689+0000] [.info] [src/director/director.go:tracef:794] [01:02:09.689] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:02:09.689+0000] [.info] [src/director/director.go:tracef:794] [01:02:09.689] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:02:09.689+0000] [.info] [src/director/director.go:tracef:794] [01:02:09.689] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:02:09.689+0000] [.info] [src/director/director.go:tracef:794] [01:02:09.689] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:02:09.690+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:02:09.690] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:02:09.840+0000] [.info] [src/director/director.go:tracef:794] [01:02:09.840] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:02:09.840+0000] [.info] [src/director/director.go:tracef:794] [01:02:09.840] because [2021-10-22T01:02:09.840+0000] [.info] [src/director/director.go:tracef:794] [01:02:09.840] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:02:09.840+0000] [.info] [src/director/director.go:tracef:794] [01:02:09.840] because ['currentState.CanBounceStop' = true] [2021-10-22T01:02:10.001+0000] [.info] [src/director/director.go:tracef:794] [01:02:10.001] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:02:10.001+0000] [.info] [src/director/director.go:tracef:794] [01:02:10.001] because [2021-10-22T01:02:10.001+0000] [.info] [src/director/director.go:tracef:794] [01:02:10.001] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:02:10.001+0000] [.info] [src/director/director.go:tracef:794] [01:02:10.001] because ['currentState.Up' = false] [2021-10-22T01:02:10.001+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:02:10.001] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:02:10.001+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:02:10.001] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:02:10.103+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:02:10.103] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:02:10.111+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:02:10.111] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:02:10.111+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:02:10.111] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:02:10.112+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:02:10.112] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:02:10.113+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:02:10.113] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:02:10.113+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:02:10.113] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:02:10.113+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:02:10.113] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:02:10.116+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:02:10.116] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:02:10.153+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:10.153] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:10.344+0000] [.info] [src/director/director.go:tracef:794] [01:02:10.344] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:02:10.344+0000] [.info] [src/director/director.go:tracef:794] [01:02:10.344] because ['currentState.Up' = false] [2021-10-22T01:02:10.577+0000] [.info] [src/director/director.go:tracef:794] [01:02:10.577] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:02:10.577+0000] [.info] [src/director/director.go:tracef:794] [01:02:10.577] because ['currentState.Up' = false] [2021-10-22T01:02:10.577+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:02:10.577] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:02:10.766+0000] [.info] [src/director/director.go:tracef:794] [01:02:10.766] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:02:10.766+0000] [.info] [src/director/director.go:tracef:794] [01:02:10.766] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:02:10.969+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:02:10.969] clusterConfig unchanged [2021-10-22T01:02:10.988+0000] [.info] [src/director/director.go:tracef:794] [01:02:10.988] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:02:10.988+0000] [.info] [src/director/director.go:tracef:794] [01:02:10.988] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:02:10.988+0000] [.info] [src/director/director.go:tracef:794] [01:02:10.988] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:02:10.988+0000] [.info] [src/director/director.go:tracef:794] [01:02:10.988] because ['currentState.Up' = false] [2021-10-22T01:02:10.988+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:02:10.988] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:02:10.988+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:02:10.988] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:02:11.024+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:02:11.024] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:02:11.025+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:02:11.025] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36834 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:02:11.145+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:11.145] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:11.740+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:02:11.740] ...Finished start [pid=3248] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36834,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:02:11.866+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:02:11.866] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 125.225494ms. [2021-10-22T01:02:11.866+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:02:11.866] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36834 (local=false) [2021-10-22T01:02:11.866+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:02:11.866] Dropping existing documents from system.replset [2021-10-22T01:02:12.065+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:02:12.065] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:02:12.069+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:02:12.069] Inserting oplogTruncateAfterPoint document [2021-10-22T01:02:12.069+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:02:12.069] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:02:12.070+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:02:12.070] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:02:12.070+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:02:12.070] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:02:12.093+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:02:12.093] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:02:12.093+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:02:12.093] Setting the minvalid document to [{_id ObjectID("61720d94523370002becc424")} {t -1} {ts {0 1}}] [2021-10-22T01:02:12.107+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:02:12.107] Inserted [{_id ObjectID("61720d94523370002becc424")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:02:12.107+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:02:12.107] Inserting new replica set configuration document. [2021-10-22T01:02:12.107+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:02:12.107] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:02:12.119+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:02:12.119] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:02:12.134+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:02:12.134] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:02:12.134+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:02:12.134] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:02:12.134+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:02:12.134] Initiating stopEphemeralMongo [2021-10-22T01:02:12.134+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:02:12.134] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36834 (local=false) [2021-10-22T01:02:12.134+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:02:12.134] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36834 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:02:12.146+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:12.146] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:12.167+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:02:12.167] clusterConfig unchanged [2021-10-22T01:02:12.549+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:02:12.549] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.112808ms. [2021-10-22T01:02:12.549+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:02:12.549] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36834 (local=false) [2021-10-22T01:02:12.549+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:02:12.549] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.733µs. [2021-10-22T01:02:12.549+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:02:12.549] Successfully finished running stopEphemeralMongo [2021-10-22T01:02:12.549+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:02:12.549] Successfully stopped existing ephemeral mongod. [2021-10-22T01:02:12.549+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:02:12.549] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:02:12.549+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:02:12.549] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:02:12.582+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:02:12.582] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:02:12.583+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:02:12.583] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42153 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:02:13.145+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:13.145] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:13.322+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:02:13.322] clusterConfig unchanged [2021-10-22T01:02:13.760+0000] [.error] [src/action/start.go:func1:93] [103] [01:02:13.760] Error starting mongod : [01:02:13.760] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42153,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42153 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 4838 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:02:12.632+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:02:12.632+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:02:12.632+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42153,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:02:12.633+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:02:12.634+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:02:12.635+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:02:12.666+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:02:12.666+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:02:12.667+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:02:12.717+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:02:12.717+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:02:13.248+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864533:248418][4838:0x7f22d421fbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:02:13.363+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864533:363690][4838:0x7f22d421fbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:02:13.471+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864533:471874][4838:0x7f22d421fbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:02:13.582+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864533:582794][4838:0x7f22d421fbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:02:13.654+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864533:654321][4838:0x7f22d421fbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:02:13.711+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864533:711366][4838:0x7f22d421fbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:02:13.711+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864533:711442][4838:0x7f22d421fbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:02:13.712+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:02:13.760+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:02:13.760] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42153 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:02:13.760] Error starting mongod : [01:02:13.760] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42153,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42153 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 4838 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:02:12.632+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:02:12.632+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:02:12.632+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42153,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:02:12.633+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:02:12.634+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:02:12.635+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:02:12.666+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:02:12.666+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:02:12.667+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:02:12.717+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:02:12.717+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:02:13.248+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864533:248418][4838:0x7f22d421fbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:02:13.363+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864533:363690][4838:0x7f22d421fbc0], txn-recover...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:02:13.760+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:02:13.760] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:02:13.760] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42153 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:02:13.760] Error starting mongod : [01:02:13.760] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42153,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42153 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 4838 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:02:12.632+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:02:12.632+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:02:12.632+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42153,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:02:12.633+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:02:12.634+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:02:12.635+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:02:12.666+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:02:12.666+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:02:12.667+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:02:12.717+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:02:12.717+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:02:13.248+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864533:248418][4838:0x7f22d421fbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:02:13.363+00:00"},"s":"I", "c":"STORAGE", "id":22430, ... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:02:13.761+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:02:13.761] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:02:13.761+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:02:13.761] Initiating stopEphemeralMongo [2021-10-22T01:02:13.761+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:02:13.761] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:02:13.763+0000] [.error] [src/director/director.go:executePlan:962] [01:02:13.763] Failed to apply action. Result = : [01:02:13.760] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:02:13.760] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42153 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:02:13.760] Error starting mongod : [01:02:13.760] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42153,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42153 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 4838 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:02:12.632+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:02:12.632+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:02:12.632+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42153,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:02:12.633+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:02:12.634+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:02:12.635+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:02:12.666+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:02:12.666+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:02:12.667+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:02:12.717+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:02:12.717+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:02:13.248+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864533:248418][4838:0x7f22d421fbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:02:13.763+0000] [.error] [src/director/director.go:planAndExecute:567] [01:02:13.763] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:02:13.763] Failed to apply action. Result = : [01:02:13.760] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:02:13.760] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42153 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:02:13.760] Error starting mongod : [01:02:13.760] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42153,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42153 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 4838 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:02:12.632+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:02:12.632+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:02:12.632+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42153,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:02:12.633+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:02:12.634+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:02:12.635+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:02:12.666+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:02:12.666+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:02:12.667+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:02:12.717+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:02:12.717+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:02:13.248+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger m... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:02:13.763+0000] [.error] [src/director/director.go:mainLoop:399] [01:02:13.763] Failed to planAndExecute : [01:02:13.763] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:02:13.763] Failed to apply action. Result = : [01:02:13.760] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:02:13.760] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42153 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:02:13.760] Error starting mongod : [01:02:13.760] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42153,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42153 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 4838 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:02:12.632+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:02:12.632+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:02:12.632+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42153,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:02:12.633+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:02:12.634+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:02:12.635+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:02:12.666+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:02:12.666+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:02:12.667+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:02:12.717+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:02:12.717+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:02:13.248+00:00"},"s":"I",... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:02:14.146+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:14.146] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:14.582+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:02:14.582] clusterConfig unchanged [2021-10-22T01:02:15.145+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:15.145] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:16.150+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:16.150] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:17.147+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:17.147] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:18.149+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:18.149] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:19.149+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:19.149] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:20.148+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:20.148] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:21.150+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:21.150] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:22.151+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:22.151] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:22.390+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:02:22.390] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.274274378s [2021-10-22T01:02:23.149+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:23.149] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:24.154+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:24.154] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:24.675+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:02:24.675] clusterConfig unchanged [2021-10-22T01:02:25.149+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:25.149] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:26.151+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:26.151] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:27.151+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:27.151] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:27.558+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:02:27.558] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:02:27.561+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:02:27.561] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:02:28.152+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:28.152] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:29.153+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:29.153] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:30.154+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:30.154] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:31.154+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:31.154] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:32.155+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:32.155] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:33.156+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:33.156] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:34.157+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:34.157] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:34.778+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:02:34.778] clusterConfig unchanged [2021-10-22T01:02:35.159+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [01:02:35.159] lsblk cache invalidated, running lsblk command [2021-10-22T01:02:35.160+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:35.160] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:36.164+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:36.164] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:37.158+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:37.158] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:38.158+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:38.158] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:39.159+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:39.159] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:40.160+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:40.160] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:41.161+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:41.161] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:41.290+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:02:41.290] memory:rss=51785728 alloc=9832184 totalalloc=3640931992 mallocs=46785691 frees=46711286 heapinuse=14450688 heapobjects=74405 gcpausetotalns=1431579097 gcpauselastns=116462 gcnum=581 utimens=64625774000 stimens=20852920000 minflt=177762 majflt=405 [2021-10-22T01:02:42.162+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:42.162] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:43.162+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:43.162] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:44.168+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:44.168] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:44.188+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [01:02:44.188] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42153 (local=false) because it is not up [2021-10-22T01:02:44.882+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:02:44.882] clusterConfig unchanged [2021-10-22T01:02:45.163+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:45.163] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:46.165+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:46.165] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:47.164+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:47.164] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:48.165+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:48.165] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:49.166+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:49.166] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:50.166+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:50.166] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:51.166+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:51.166] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:52.166+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:52.166] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:53.172+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:53.172] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:54.167+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:54.167] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:54.991+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:02:54.991] clusterConfig unchanged [2021-10-22T01:02:55.168+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:55.168] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:56.168+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:56.168] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:57.169+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:57.169] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:58.169+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:58.169] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:02:59.170+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:02:59.170] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:00.170+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:00.170] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:01.171+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:01.171] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:02.172+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:02.172] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:03.177+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:03.177] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:04.173+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:04.173] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:05.113+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:03:05.113] clusterConfig unchanged [2021-10-22T01:03:05.174+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:05.174] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:06.177+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:06.177] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:07.175+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:07.175] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:08.176+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:08.176] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:09.176+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:09.176] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:10.177+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:10.177] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:11.178+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:11.178] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:12.178+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:12.178] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:13.179+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:13.179] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:13.931+0000] [.info] [src/director/director.go:computePlan:280] [01:03:13.931] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:03:13.932+0000] [.info] [src/director/director.go:tracef:794] [01:03:13.931] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:03:13.932+0000] [.info] [src/director/director.go:tracef:794] [01:03:13.932] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:03:13.932+0000] [.info] [src/director/director.go:tracef:794] [01:03:13.932] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:03:13.932+0000] [.info] [src/director/director.go:tracef:794] [01:03:13.932] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:03:13.932+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:03:13.932] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:03:14.094+0000] [.info] [src/director/director.go:tracef:794] [01:03:14.094] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:03:14.094+0000] [.info] [src/director/director.go:tracef:794] [01:03:14.094] because [2021-10-22T01:03:14.094+0000] [.info] [src/director/director.go:tracef:794] [01:03:14.094] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:03:14.094+0000] [.info] [src/director/director.go:tracef:794] [01:03:14.094] because ['currentState.CanBounceStop' = true] [2021-10-22T01:03:14.182+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:14.182] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:14.262+0000] [.info] [src/director/director.go:tracef:794] [01:03:14.262] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:03:14.262+0000] [.info] [src/director/director.go:tracef:794] [01:03:14.262] because [2021-10-22T01:03:14.262+0000] [.info] [src/director/director.go:tracef:794] [01:03:14.262] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:03:14.262+0000] [.info] [src/director/director.go:tracef:794] [01:03:14.262] because ['currentState.Up' = false] [2021-10-22T01:03:14.262+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:03:14.262] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:03:14.262+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:03:14.262] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:03:14.339+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:03:14.339] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:03:14.347+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:03:14.347] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:03:14.348+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:03:14.348] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:03:14.349+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:03:14.349] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:03:14.350+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:03:14.350] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:03:14.350+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:03:14.350] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:03:14.350+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:03:14.350] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:03:14.351+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:03:14.351] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:03:14.565+0000] [.info] [src/director/director.go:tracef:794] [01:03:14.565] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:03:14.565+0000] [.info] [src/director/director.go:tracef:794] [01:03:14.565] because ['currentState.Up' = false] [2021-10-22T01:03:14.792+0000] [.info] [src/director/director.go:tracef:794] [01:03:14.792] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:03:14.792+0000] [.info] [src/director/director.go:tracef:794] [01:03:14.792] because ['currentState.Up' = false] [2021-10-22T01:03:14.792+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:03:14.792] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:03:14.983+0000] [.info] [src/director/director.go:tracef:794] [01:03:14.983] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:03:14.984+0000] [.info] [src/director/director.go:tracef:794] [01:03:14.984] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:03:15.182+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:15.182] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:15.202+0000] [.info] [src/director/director.go:tracef:794] [01:03:15.202] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:03:15.202+0000] [.info] [src/director/director.go:tracef:794] [01:03:15.202] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:03:15.203+0000] [.info] [src/director/director.go:tracef:794] [01:03:15.203] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:03:15.203+0000] [.info] [src/director/director.go:tracef:794] [01:03:15.203] because ['currentState.Up' = false] [2021-10-22T01:03:15.203+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:03:15.203] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:03:15.203+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:03:15.203] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:03:15.238+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:03:15.238] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:03:15.238+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:03:15.238] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45840 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:03:15.294+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:03:15.294] clusterConfig unchanged [2021-10-22T01:03:15.955+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:03:15.955] ...Finished start [pid=23972] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45840,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:03:16.091+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:03:16.091] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 136.034758ms. [2021-10-22T01:03:16.091+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:03:16.091] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45840 (local=false) [2021-10-22T01:03:16.091+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:03:16.091] Dropping existing documents from system.replset [2021-10-22T01:03:16.184+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:16.184] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:16.234+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:03:16.234] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:03:16.244+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:03:16.244] Inserting oplogTruncateAfterPoint document [2021-10-22T01:03:16.244+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:03:16.244] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:03:16.245+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:03:16.245] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:03:16.245+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:03:16.245] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:03:16.259+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:03:16.258] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:03:16.259+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:03:16.259] Setting the minvalid document to [{_id ObjectID("61720dd4523370002becc432")} {t -1} {ts {0 1}}] [2021-10-22T01:03:16.271+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:03:16.271] Inserted [{_id ObjectID("61720dd4523370002becc432")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:03:16.272+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:03:16.272] Inserting new replica set configuration document. [2021-10-22T01:03:16.272+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:03:16.272] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:03:16.285+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:03:16.285] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:03:16.288+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:03:16.288] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:03:16.288+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:03:16.288] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:03:16.288+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:03:16.288] Initiating stopEphemeralMongo [2021-10-22T01:03:16.288+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:03:16.288] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45840 (local=false) [2021-10-22T01:03:16.288+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:03:16.288] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45840 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:03:16.434+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:03:16.434] clusterConfig unchanged [2021-10-22T01:03:16.765+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:03:16.765] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.105687ms. [2021-10-22T01:03:16.765+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:03:16.765] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45840 (local=false) [2021-10-22T01:03:16.765+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:03:16.765] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.934µs. [2021-10-22T01:03:16.765+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:03:16.765] Successfully finished running stopEphemeralMongo [2021-10-22T01:03:16.765+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:03:16.765] Successfully stopped existing ephemeral mongod. [2021-10-22T01:03:16.765+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:03:16.765] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:03:16.765+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:03:16.765] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:03:16.799+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:03:16.799] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:03:16.800+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:03:16.800] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37124 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:03:17.184+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:17.184] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:17.549+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:03:17.549] clusterConfig unchanged [2021-10-22T01:03:17.998+0000] [.error] [src/action/start.go:func1:93] [103] [01:03:17.998] Error starting mongod : [01:03:17.998] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37124,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37124 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:03:16.832Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 25562 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:03:16.852+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:03:16.852+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:03:16.852+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37124,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:03:16.854+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:03:16.856+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:03:16.857+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:03:16.933+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:03:16.933+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:03:16.933+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:03:16.989+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:03:16.989+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:03:17.482+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864597:482788][25562:0x7fb0c5c62bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:03:17.599+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864597:599565][25562:0x7fb0c5c62bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:03:17.704+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864597:704847][25562:0x7fb0c5c62bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:03:17.818+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864597:818453][25562:0x7fb0c5c62bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:03:17.893+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864597:893418][25562:0x7fb0c5c62bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:03:17.950+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864597:950663][25562:0x7fb0c5c62bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:03:17.950+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864597:950749][25562:0x7fb0c5c62bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:03:17.998+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:03:17.998] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37124 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:03:17.998] Error starting mongod : [01:03:17.998] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37124,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37124 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:03:16.832Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 25562 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:03:16.852+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:03:16.852+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:03:16.852+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37124,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:03:16.854+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:03:16.856+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:03:16.857+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:03:16.933+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:03:16.933+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:03:16.933+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:03:16.989+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:03:16.989+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:03:17.482+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864597:482788][25562:0x7fb0c5c62bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:03:17.599+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:03:17.999+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:03:17.999] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:03:17.998] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37124 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:03:17.998] Error starting mongod : [01:03:17.998] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37124,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37124 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:03:16.832Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 25562 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:03:16.852+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:03:16.852+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:03:16.852+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37124,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:03:16.854+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:03:16.856+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:03:16.857+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:03:16.933+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:03:16.933+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:03:16.933+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:03:16.989+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:03:16.989+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:03:17.482+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864597:482788][25562:0x7fb0c5c62bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:03:17.999+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:03:17.999] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:03:17.999+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:03:17.999] Initiating stopEphemeralMongo [2021-10-22T01:03:17.999+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:03:17.999] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:03:17.999+0000] [.error] [src/director/director.go:executePlan:962] [01:03:17.999] Failed to apply action. Result = : [01:03:17.999] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:03:17.998] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37124 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:03:17.998] Error starting mongod : [01:03:17.998] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37124,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37124 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:03:16.832Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 25562 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:03:16.852+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:03:16.852+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:03:16.852+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37124,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:03:16.854+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:03:16.856+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:03:16.857+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:03:16.933+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:03:16.933+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:03:16.933+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:03:16.989+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:03:16.989+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:03:17.482+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:03:17.999+0000] [.error] [src/director/director.go:planAndExecute:567] [01:03:17.999] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:03:17.999] Failed to apply action. Result = : [01:03:17.999] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:03:17.998] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37124 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:03:17.998] Error starting mongod : [01:03:17.998] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37124,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37124 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:03:16.832Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 25562 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:03:16.852+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:03:16.852+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:03:16.852+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37124,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:03:16.854+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:03:16.856+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:03:16.857+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:03:16.933+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:03:16.933+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:03:16.933+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:03:16.989+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:03:16.989+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:03:18.000+0000] [.error] [src/director/director.go:mainLoop:399] [01:03:18.000] Failed to planAndExecute : [01:03:17.999] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:03:17.999] Failed to apply action. Result = : [01:03:17.999] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:03:17.998] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37124 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:03:17.998] Error starting mongod : [01:03:17.998] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37124,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37124 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:03:16.832Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 25562 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:03:16.852+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:03:16.852+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:03:16.852+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37124,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:03:16.854+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:03:16.856+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:03:16.857+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:03:16.933+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:03:16.933+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:03:16.933+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:03:16.989+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:03:16.989+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:03:18.184+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:18.184] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:18.789+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:03:18.789] clusterConfig unchanged [2021-10-22T01:03:19.187+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:19.187] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:20.190+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:20.190] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:21.185+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:21.185] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:22.186+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:22.186] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:23.188+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:23.188] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:24.187+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:24.187] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:25.187+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:25.187] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:26.187+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:26.187] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:26.740+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:03:26.740] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.388709691s [2021-10-22T01:03:27.210+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:27.210] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:28.196+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:28.196] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:28.911+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:03:28.911] clusterConfig unchanged [2021-10-22T01:03:29.197+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:29.197] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:30.198+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:30.198] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:31.210+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:31.210] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:32.195+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:32.195] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:33.197+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:33.197] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:34.199+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:34.199] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:35.199+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:35.199] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:36.198+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:36.198] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:37.198+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:37.198] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:38.200+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:38.200] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:39.013+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:03:39.013] clusterConfig unchanged [2021-10-22T01:03:39.198+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:39.198] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:40.198+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:40.198] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:41.201+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:41.201] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:41.291+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:03:41.291] memory:rss=52043776 alloc=12428720 totalalloc=3727789480 mallocs=47875851 frees=47773997 heapinuse=16007168 heapobjects=101854 gcpausetotalns=1437407583 gcpauselastns=122881 gcnum=594 utimens=66551942000 stimens=21657616000 minflt=182531 majflt=405 [2021-10-22T01:03:42.200+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:42.200] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:43.199+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:43.199] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:44.201+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:44.201] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:45.202+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:45.202] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:46.203+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:46.203] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:47.200+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:47.200] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:48.201+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:48.201] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:49.115+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:03:49.115] clusterConfig unchanged [2021-10-22T01:03:49.202+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:49.202] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:50.202+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:50.202] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:51.203+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:51.203] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:52.203+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:52.203] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:53.204+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:53.204] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:54.210+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:54.210] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:55.205+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:55.205] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:56.206+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:56.206] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:57.207+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:57.207] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:58.207+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:58.207] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:59.208+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:03:59.208] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:03:59.227+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:03:59.227] clusterConfig unchanged [2021-10-22T01:04:00.208+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:00.208] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:01.209+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:01.209] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:02.213+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:02.213] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:03.211+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:03.211] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:04.211+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:04.211] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:05.217+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:05.217] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:06.213+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:06.213] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:07.213+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:07.213] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:08.213+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:08.213] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:09.214+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:09.214] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:09.339+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:04:09.339] clusterConfig unchanged [2021-10-22T01:04:10.215+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:10.215] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:11.219+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:11.219] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:12.217+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:12.217] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:13.217+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:13.217] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:14.217+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:14.217] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:15.218+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:15.218] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:16.219+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:16.219] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:17.219+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:17.219] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:18.168+0000] [.info] [src/director/director.go:computePlan:280] [01:04:18.168] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:04:18.168+0000] [.info] [src/director/director.go:tracef:794] [01:04:18.168] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:04:18.168+0000] [.info] [src/director/director.go:tracef:794] [01:04:18.168] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:04:18.168+0000] [.info] [src/director/director.go:tracef:794] [01:04:18.168] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:04:18.168+0000] [.info] [src/director/director.go:tracef:794] [01:04:18.168] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:04:18.168+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:04:18.168] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:04:18.232+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:18.232] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:18.325+0000] [.info] [src/director/director.go:tracef:794] [01:04:18.325] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:04:18.325+0000] [.info] [src/director/director.go:tracef:794] [01:04:18.325] because [2021-10-22T01:04:18.326+0000] [.info] [src/director/director.go:tracef:794] [01:04:18.326] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:04:18.326+0000] [.info] [src/director/director.go:tracef:794] [01:04:18.326] because ['currentState.CanBounceStop' = true] [2021-10-22T01:04:18.477+0000] [.info] [src/director/director.go:tracef:794] [01:04:18.477] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:04:18.477+0000] [.info] [src/director/director.go:tracef:794] [01:04:18.477] because [2021-10-22T01:04:18.477+0000] [.info] [src/director/director.go:tracef:794] [01:04:18.477] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:04:18.477+0000] [.info] [src/director/director.go:tracef:794] [01:04:18.477] because ['currentState.Up' = false] [2021-10-22T01:04:18.477+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:04:18.477] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:04:18.477+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:04:18.477] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:04:18.625+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:04:18.625] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:04:18.633+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:04:18.633] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:04:18.634+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:04:18.634] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:04:18.635+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:04:18.635] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:04:18.636+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:04:18.636] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:04:18.636+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:04:18.636] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:04:18.636+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:04:18.636] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:04:18.637+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:04:18.637] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:04:18.848+0000] [.info] [src/director/director.go:tracef:794] [01:04:18.848] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:04:18.848+0000] [.info] [src/director/director.go:tracef:794] [01:04:18.848] because ['currentState.Up' = false] [2021-10-22T01:04:19.060+0000] [.info] [src/director/director.go:tracef:794] [01:04:19.060] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:04:19.060+0000] [.info] [src/director/director.go:tracef:794] [01:04:19.060] because ['currentState.Up' = false] [2021-10-22T01:04:19.061+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:04:19.061] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:04:19.222+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:19.222] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:19.263+0000] [.info] [src/director/director.go:tracef:794] [01:04:19.263] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:04:19.263+0000] [.info] [src/director/director.go:tracef:794] [01:04:19.263] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:04:19.465+0000] [.info] [src/director/director.go:tracef:794] [01:04:19.465] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:04:19.465+0000] [.info] [src/director/director.go:tracef:794] [01:04:19.465] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:04:19.465+0000] [.info] [src/director/director.go:tracef:794] [01:04:19.465] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:04:19.465+0000] [.info] [src/director/director.go:tracef:794] [01:04:19.465] because ['currentState.Up' = false] [2021-10-22T01:04:19.465+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:04:19.465] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:04:19.465+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:04:19.465] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:04:19.499+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:04:19.499] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:04:19.500+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:04:19.500] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37963 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:04:19.521+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:04:19.521] clusterConfig unchanged [2021-10-22T01:04:20.223+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:20.222] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:20.233+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:04:20.233] ...Finished start [pid=45238] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37963,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:04:20.353+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:04:20.353] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 119.993019ms. [2021-10-22T01:04:20.353+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:04:20.353] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37963 (local=false) [2021-10-22T01:04:20.353+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:04:20.353] Dropping existing documents from system.replset [2021-10-22T01:04:20.475+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:04:20.475] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:04:20.480+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:04:20.480] Inserting oplogTruncateAfterPoint document [2021-10-22T01:04:20.480+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:04:20.480] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:04:20.481+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:04:20.481] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:04:20.481+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:04:20.481] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:04:20.494+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:04:20.494] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:04:20.494+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:04:20.494] Setting the minvalid document to [{_id ObjectID("61720e14523370002becc440")} {t -1} {ts {0 1}}] [2021-10-22T01:04:20.507+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:04:20.507] Inserted [{_id ObjectID("61720e14523370002becc440")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:04:20.507+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:04:20.507] Inserting new replica set configuration document. [2021-10-22T01:04:20.507+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:04:20.507] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:04:20.519+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:04:20.519] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:04:20.523+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:04:20.523] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:04:20.523+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:04:20.523] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:04:20.523+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:04:20.523] Initiating stopEphemeralMongo [2021-10-22T01:04:20.523+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:04:20.523] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37963 (local=false) [2021-10-22T01:04:20.523+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:04:20.523] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37963 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:04:20.643+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:04:20.643] clusterConfig unchanged [2021-10-22T01:04:21.010+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:04:21.009] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.174085ms. [2021-10-22T01:04:21.010+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:04:21.010] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37963 (local=false) [2021-10-22T01:04:21.010+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:04:21.010] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.948µs. [2021-10-22T01:04:21.010+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:04:21.010] Successfully finished running stopEphemeralMongo [2021-10-22T01:04:21.010+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:04:21.010] Successfully stopped existing ephemeral mongod. [2021-10-22T01:04:21.010+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:04:21.010] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:04:21.010+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:04:21.010] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:04:21.048+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:04:21.048] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:04:21.049+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:04:21.049] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36834 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:04:21.237+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:21.237] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:21.775+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:04:21.775] clusterConfig unchanged [2021-10-22T01:04:22.238+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:22.238] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:22.299+0000] [.error] [src/action/start.go:func1:93] [103] [01:04:22.299] Error starting mongod : [01:04:22.299] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36834,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36834 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 46669 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:04:21.101+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:04:21.101+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:04:21.101+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36834,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:04:21.104+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:04:21.106+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:04:21.106+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:04:21.127+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:04:21.127+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:04:21.138+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:04:21.182+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:04:21.182+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:04:21.803+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864661:803752][46669:0x7f8d27ec2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:04:21.900+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864661:900876][46669:0x7f8d27ec2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:04:21.999+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864661:999450][46669:0x7f8d27ec2bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35456 to 2/256"}} {"t":{"$date":"2021-10-22T01:04:22.114+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864662:114098][46669:0x7f8d27ec2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:04:22.184+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864662:184907][46669:0x7f8d27ec2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:04:22.244+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864662:244818][46669:0x7f8d27ec2bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:04:22.244+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864662:244885][46669:0x7f8d27ec2bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:04:22.246+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:04:22.299+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:04:22.299] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36834 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:04:22.299] Error starting mongod : [01:04:22.299] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36834,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36834 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 46669 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:04:21.101+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:04:21.101+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:04:21.101+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36834,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:04:21.104+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:04:21.106+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:04:21.106+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:04:21.127+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:04:21.127+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:04:21.138+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:04:21.182+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:04:21.182+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:04:21.803+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864661:803752][46669:0x7f8d27ec2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:04:21.900+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864661:900876][46669:0x7f8d27ec2bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:04:22.299+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:04:22.299] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:04:22.299] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36834 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:04:22.299] Error starting mongod : [01:04:22.299] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36834,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36834 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 46669 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:04:21.101+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:04:21.101+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:04:21.101+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36834,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:04:21.104+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:04:21.106+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:04:21.106+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:04:21.127+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:04:21.127+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:04:21.138+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:04:21.182+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:04:21.182+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:04:21.803+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864661:803752][46669:0x7f8d27ec2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:04:21.900+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:04:22.299+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:04:22.299] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:04:22.299+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:04:22.299] Initiating stopEphemeralMongo [2021-10-22T01:04:22.299+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:04:22.299] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:04:22.302+0000] [.error] [src/director/director.go:executePlan:962] [01:04:22.302] Failed to apply action. Result = : [01:04:22.299] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:04:22.299] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36834 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:04:22.299] Error starting mongod : [01:04:22.299] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36834,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36834 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 46669 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:04:21.101+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:04:21.101+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:04:21.101+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36834,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:04:21.104+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:04:21.106+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:04:21.106+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:04:21.127+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:04:21.127+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:04:21.138+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:04:21.182+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:04:21.182+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:04:21.803+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864661:803752][46669:0x7f8d27ec2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:04:22.303+0000] [.error] [src/director/director.go:planAndExecute:567] [01:04:22.303] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:04:22.302] Failed to apply action. Result = : [01:04:22.299] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:04:22.299] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36834 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:04:22.299] Error starting mongod : [01:04:22.299] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36834,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36834 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 46669 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:04:21.101+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:04:21.101+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:04:21.101+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36834,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:04:21.104+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:04:21.106+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:04:21.106+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:04:21.127+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:04:21.127+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:04:21.138+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:04:21.182+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:04:21.182+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:04:21.803+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:04:22.303+0000] [.error] [src/director/director.go:mainLoop:399] [01:04:22.303] Failed to planAndExecute : [01:04:22.303] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:04:22.302] Failed to apply action. Result = : [01:04:22.299] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:04:22.299] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36834 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:04:22.299] Error starting mongod : [01:04:22.299] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36834,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36834 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 46669 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:04:21.101+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:04:21.101+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:04:21.101+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36834,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:04:21.104+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:04:21.106+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:04:21.106+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:04:21.127+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:04:21.127+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:04:21.138+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:04:21.182+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:04:21.182+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:04:21.803+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:04:23.026+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:04:23.026] clusterConfig unchanged [2021-10-22T01:04:23.237+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:23.237] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:24.238+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:24.238] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:25.241+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:25.241] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:26.240+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:26.240] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:27.244+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:27.244] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:28.246+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:28.246] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:29.246+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:29.246] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:30.247+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:30.247] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:30.967+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:04:30.967] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.329893792s [2021-10-22T01:04:31.244+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:31.244] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:32.244+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:32.244] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:33.137+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:04:33.137] clusterConfig unchanged [2021-10-22T01:04:33.246+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:33.246] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:34.246+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:34.246] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:35.247+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:35.247] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:36.247+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:36.247] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:37.248+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:37.248] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:38.248+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:38.248] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:39.249+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:39.249] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:40.249+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:40.249] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:41.270+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:41.270] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:41.292+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:04:41.292] memory:rss=51675136 alloc=7273264 totalalloc=3813294320 mallocs=48951885 frees=48905890 heapinuse=14295040 heapobjects=45995 gcpausetotalns=1441189189 gcpauselastns=203351 gcnum=608 utimens=68340015000 stimens=22386452000 minflt=186857 majflt=405 [2021-10-22T01:04:42.250+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:42.250] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:43.250+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:04:43.250] clusterConfig unchanged [2021-10-22T01:04:43.250+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:43.250] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:44.251+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:44.251] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:45.251+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:45.251] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:46.252+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:46.252] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:47.254+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:47.254] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:48.253+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:48.253] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:49.256+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:49.256] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:50.255+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:50.255] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:51.256+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:51.256] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:52.256+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:52.256] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:53.256+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:53.256] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:53.362+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:04:53.362] clusterConfig unchanged [2021-10-22T01:04:54.257+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:54.257] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:55.258+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:55.258] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:56.259+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:56.259] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:57.259+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:57.259] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:58.260+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:58.260] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:04:59.260+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:04:59.260] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:00.281+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:00.281] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:01.262+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:01.262] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:02.264+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:02.264] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:03.263+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:03.263] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:03.503+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:05:03.503] clusterConfig unchanged [2021-10-22T01:05:04.264+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:04.264] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:05.265+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:05.265] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:06.266+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:06.266] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:07.267+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:07.267] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:08.267+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:08.267] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:09.268+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:09.268] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:10.269+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:10.269] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:11.275+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:11.275] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:12.271+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:12.271] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:13.271+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:13.271] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:13.635+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:05:13.635] clusterConfig unchanged [2021-10-22T01:05:14.272+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:14.272] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:15.273+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:15.273] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:16.273+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:16.273] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:17.274+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:17.274] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:18.277+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:18.277] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:19.275+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:19.275] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:20.276+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:20.276] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:21.277+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:21.277] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:22.283+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:22.283] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:22.475+0000] [.info] [src/director/director.go:computePlan:280] [01:05:22.475] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:05:22.475+0000] [.info] [src/director/director.go:tracef:794] [01:05:22.475] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:05:22.475+0000] [.info] [src/director/director.go:tracef:794] [01:05:22.475] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:05:22.475+0000] [.info] [src/director/director.go:tracef:794] [01:05:22.475] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:05:22.475+0000] [.info] [src/director/director.go:tracef:794] [01:05:22.475] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:05:22.476+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:05:22.476] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:05:22.646+0000] [.info] [src/director/director.go:tracef:794] [01:05:22.646] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:05:22.646+0000] [.info] [src/director/director.go:tracef:794] [01:05:22.646] because [2021-10-22T01:05:22.646+0000] [.info] [src/director/director.go:tracef:794] [01:05:22.646] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:05:22.646+0000] [.info] [src/director/director.go:tracef:794] [01:05:22.646] because ['currentState.CanBounceStop' = true] [2021-10-22T01:05:22.798+0000] [.info] [src/director/director.go:tracef:794] [01:05:22.798] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:05:22.798+0000] [.info] [src/director/director.go:tracef:794] [01:05:22.798] because [2021-10-22T01:05:22.798+0000] [.info] [src/director/director.go:tracef:794] [01:05:22.798] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:05:22.798+0000] [.info] [src/director/director.go:tracef:794] [01:05:22.798] because ['currentState.Up' = false] [2021-10-22T01:05:22.798+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:05:22.798] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:05:22.798+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:05:22.798] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:05:22.881+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:05:22.881] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:05:22.881+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:05:22.881] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:05:22.889+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:05:22.889] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:05:22.894+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:05:22.894] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:05:22.894+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:05:22.894] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:05:22.895+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:05:22.895] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:05:22.897+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:05:22.897] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:05:23.111+0000] [.info] [src/director/director.go:tracef:794] [01:05:23.111] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:05:23.111+0000] [.info] [src/director/director.go:tracef:794] [01:05:23.111] because ['currentState.Up' = false] [2021-10-22T01:05:23.280+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:23.280] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:23.320+0000] [.info] [src/director/director.go:tracef:794] [01:05:23.320] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:05:23.320+0000] [.info] [src/director/director.go:tracef:794] [01:05:23.320] because ['currentState.Up' = false] [2021-10-22T01:05:23.320+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:05:23.320] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:05:23.519+0000] [.info] [src/director/director.go:tracef:794] [01:05:23.519] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:05:23.519+0000] [.info] [src/director/director.go:tracef:794] [01:05:23.519] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:05:23.776+0000] [.info] [src/director/director.go:tracef:794] [01:05:23.776] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:05:23.777+0000] [.info] [src/director/director.go:tracef:794] [01:05:23.777] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:05:23.777+0000] [.info] [src/director/director.go:tracef:794] [01:05:23.777] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:05:23.777+0000] [.info] [src/director/director.go:tracef:794] [01:05:23.777] because ['currentState.Up' = false] [2021-10-22T01:05:23.777+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:05:23.777] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:05:23.777+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:05:23.777] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:05:23.814+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:05:23.814] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:05:23.815+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:05:23.815] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49272 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:05:23.855+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:05:23.855] clusterConfig unchanged [2021-10-22T01:05:24.284+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:24.284] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:24.532+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:05:24.532] ...Finished start [pid=66032] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49272,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:05:24.660+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:05:24.660] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 127.760915ms. [2021-10-22T01:05:24.660+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:05:24.660] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49272 (local=false) [2021-10-22T01:05:24.660+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:05:24.660] Dropping existing documents from system.replset [2021-10-22T01:05:24.802+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:05:24.802] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:05:24.808+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:05:24.808] Inserting oplogTruncateAfterPoint document [2021-10-22T01:05:24.808+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:05:24.808] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:05:24.810+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:05:24.810] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:05:24.810+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:05:24.810] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:05:24.823+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:05:24.823] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:05:24.823+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:05:24.823] Setting the minvalid document to [{_id ObjectID("61720e54523370002becc44e")} {t -1} {ts {0 1}}] [2021-10-22T01:05:24.835+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:05:24.835] Inserted [{_id ObjectID("61720e54523370002becc44e")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:05:24.835+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:05:24.835] Inserting new replica set configuration document. [2021-10-22T01:05:24.835+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:05:24.835] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:05:24.847+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:05:24.847] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:05:24.851+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:05:24.851] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:05:24.851+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:05:24.851] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:05:24.851+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:05:24.851] Initiating stopEphemeralMongo [2021-10-22T01:05:24.851+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:05:24.851] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49272 (local=false) [2021-10-22T01:05:24.851+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:05:24.851] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49272 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:05:24.970+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:05:24.969] clusterConfig unchanged [2021-10-22T01:05:25.284+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:25.284] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:25.307+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:05:25.307] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.193003ms. [2021-10-22T01:05:25.307+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:05:25.307] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49272 (local=false) [2021-10-22T01:05:25.307+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:05:25.307] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 39.012µs. [2021-10-22T01:05:25.307+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:05:25.307] Successfully finished running stopEphemeralMongo [2021-10-22T01:05:25.307+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:05:25.307] Successfully stopped existing ephemeral mongod. [2021-10-22T01:05:25.307+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:05:25.307] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:05:25.307+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:05:25.307] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:05:25.343+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:05:25.343] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:05:25.344+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:05:25.344] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38098 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:05:26.082+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:05:26.082] clusterConfig unchanged [2021-10-22T01:05:26.308+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:26.308] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:26.517+0000] [.error] [src/action/start.go:func1:93] [103] [01:05:26.517] Error starting mongod : [01:05:26.517] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38098,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38098 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 67672 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:05:25.393+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:05:25.393+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:05:25.393+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38098,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:05:25.395+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:05:25.396+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:05:25.414+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:05:25.432+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:05:25.432+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:05:25.432+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:05:25.494+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:05:25.494+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:05:26.003+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864726:3536][67672:0x7f70a6f09bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:05:26.109+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864726:109783][67672:0x7f70a6f09bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:05:26.213+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864726:213454][67672:0x7f70a6f09bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:05:26.333+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864726:333817][67672:0x7f70a6f09bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:05:26.406+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864726:406879][67672:0x7f70a6f09bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:05:26.465+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864726:465845][67672:0x7f70a6f09bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:05:26.465+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864726:465919][67672:0x7f70a6f09bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:05:26.467+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:05:26.517+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:05:26.517] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38098 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:05:26.517] Error starting mongod : [01:05:26.517] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38098,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38098 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 67672 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:05:25.393+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:05:25.393+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:05:25.393+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38098,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:05:25.395+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:05:25.396+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:05:25.414+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:05:25.432+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:05:25.432+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:05:25.432+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:05:25.494+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:05:25.494+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:05:26.003+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864726:3536][67672:0x7f70a6f09bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:05:26.109+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864726:109783][67672:0x7f70a6f09bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:05:26.517+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:05:26.517] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:05:26.517] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38098 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:05:26.517] Error starting mongod : [01:05:26.517] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38098,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38098 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 67672 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:05:25.393+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:05:25.393+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:05:25.393+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38098,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:05:25.395+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:05:25.396+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:05:25.414+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:05:25.432+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:05:25.432+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:05:25.432+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:05:25.494+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:05:25.494+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:05:26.003+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864726:3536][67672:0x7f70a6f09bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:05:26.109+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:05:26.517+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:05:26.517] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:05:26.517+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:05:26.517] Initiating stopEphemeralMongo [2021-10-22T01:05:26.517+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:05:26.517] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:05:26.518+0000] [.error] [src/director/director.go:executePlan:962] [01:05:26.518] Failed to apply action. Result = : [01:05:26.517] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:05:26.517] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38098 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:05:26.517] Error starting mongod : [01:05:26.517] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38098,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38098 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 67672 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:05:25.393+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:05:25.393+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:05:25.393+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38098,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:05:25.395+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:05:25.396+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:05:25.414+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:05:25.432+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:05:25.432+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:05:25.432+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:05:25.494+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:05:25.494+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:05:26.003+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864726:3536][67672:0x7f70a6f09bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:05:26.518+0000] [.error] [src/director/director.go:planAndExecute:567] [01:05:26.518] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:05:26.518] Failed to apply action. Result = : [01:05:26.517] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:05:26.517] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38098 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:05:26.517] Error starting mongod : [01:05:26.517] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38098,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38098 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 67672 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:05:25.393+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:05:25.393+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:05:25.393+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38098,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:05:25.395+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:05:25.396+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:05:25.414+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:05:25.432+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:05:25.432+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:05:25.432+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:05:25.494+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:05:25.494+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:05:26.003+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger messag... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:05:26.518+0000] [.error] [src/director/director.go:mainLoop:399] [01:05:26.518] Failed to planAndExecute : [01:05:26.518] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:05:26.518] Failed to apply action. Result = : [01:05:26.517] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:05:26.517] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38098 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:05:26.517] Error starting mongod : [01:05:26.517] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38098,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38098 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 67672 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:05:25.393+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:05:25.393+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:05:25.393+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38098,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:05:25.395+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:05:25.396+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:05:25.414+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:05:25.432+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:05:25.432+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:05:25.432+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:05:25.494+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:05:25.494+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:05:26.003+00:00"},"s":"I", "c"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:05:27.292+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:27.292] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:27.321+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:05:27.321] clusterConfig unchanged [2021-10-22T01:05:28.293+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:28.293] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:29.293+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:29.293] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:30.292+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:30.292] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:31.295+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:31.295] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:32.296+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:32.296] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:33.296+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:33.296] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:34.297+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:34.296] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:35.270+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:05:35.270] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.372711983s [2021-10-22T01:05:35.299+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:35.299] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:36.294+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:36.294] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:37.295+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:37.295] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:37.423+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:05:37.423] clusterConfig unchanged [2021-10-22T01:05:38.295+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:38.295] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:39.297+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:39.297] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:40.296+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:40.296] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:41.292+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:05:41.292] memory:rss=51273728 alloc=9859168 totalalloc=3897925760 mallocs=50034152 frees=49958838 heapinuse=14802944 heapobjects=75314 gcpausetotalns=1442600334 gcpauselastns=53480 gcnum=621 utimens=69935069000 stimens=23003743000 minflt=191119 majflt=405 [2021-10-22T01:05:41.297+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:41.297] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:42.297+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:42.297] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:43.298+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:43.298] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:44.200+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [01:05:44.200] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38098 (local=false) because it is not up [2021-10-22T01:05:44.299+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:44.298] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:45.299+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:45.299] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:46.300+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:46.300] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:47.300+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:47.300] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:47.555+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:05:47.555] clusterConfig unchanged [2021-10-22T01:05:48.301+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:48.301] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:49.301+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:49.301] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:50.301+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:50.301] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:51.303+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:51.303] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:52.304+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:52.304] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:53.304+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:53.304] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:54.304+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:54.304] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:55.305+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:55.305] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:56.322+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:56.322] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:57.306+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:57.306] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:57.676+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:05:57.676] clusterConfig unchanged [2021-10-22T01:05:58.307+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:58.307] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:05:59.307+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:05:59.307] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:00.308+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:00.308] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:01.310+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:01.310] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:02.309+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:02.309] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:03.309+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:03.309] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:04.315+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:04.315] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:05.310+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:05.310] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:06.311+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:06.311] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:07.311+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:07.311] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:07.791+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:06:07.791] clusterConfig unchanged [2021-10-22T01:06:08.313+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:08.313] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:09.313+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:09.313] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:10.319+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:10.319] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:11.315+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:11.315] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:12.315+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:12.315] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:13.316+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:13.316] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:14.320+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:14.320] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:15.323+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:15.323] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:16.318+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:16.318] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:17.321+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:17.321] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:17.881+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:06:17.881] clusterConfig unchanged [2021-10-22T01:06:18.320+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:18.320] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:18.558+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:06:18.557] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:06:18.560+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:18.560] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:06:18.560+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [01:06:18.560] Could not get version : [01:06:18.560] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:06:18.560+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [01:06:18.560] Error getting version for log lines. Skipping : [01:06:18.560] Could not get version : [01:06:18.560] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:06:19.320+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:19.320] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:20.320+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:20.320] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:21.321+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:21.321] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:22.321+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:22.321] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:23.322+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:23.322] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:24.328+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:24.328] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:25.323+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:25.323] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:26.326+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:26.326] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:26.790+0000] [.info] [src/director/director.go:computePlan:280] [01:06:26.790] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:06:26.790+0000] [.info] [src/director/director.go:tracef:794] [01:06:26.790] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:06:26.790+0000] [.info] [src/director/director.go:tracef:794] [01:06:26.790] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:06:26.790+0000] [.info] [src/director/director.go:tracef:794] [01:06:26.790] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:06:26.790+0000] [.info] [src/director/director.go:tracef:794] [01:06:26.790] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:06:26.791+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:06:26.790] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:06:26.993+0000] [.info] [src/director/director.go:tracef:794] [01:06:26.993] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:06:26.994+0000] [.info] [src/director/director.go:tracef:794] [01:06:26.994] because [2021-10-22T01:06:26.994+0000] [.info] [src/director/director.go:tracef:794] [01:06:26.994] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:06:26.994+0000] [.info] [src/director/director.go:tracef:794] [01:06:26.994] because ['currentState.CanBounceStop' = true] [2021-10-22T01:06:27.226+0000] [.info] [src/director/director.go:tracef:794] [01:06:27.226] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:06:27.226+0000] [.info] [src/director/director.go:tracef:794] [01:06:27.226] because [2021-10-22T01:06:27.226+0000] [.info] [src/director/director.go:tracef:794] [01:06:27.226] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:06:27.226+0000] [.info] [src/director/director.go:tracef:794] [01:06:27.226] because ['currentState.Up' = false] [2021-10-22T01:06:27.226+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:06:27.226] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:06:27.226+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:06:27.226] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:06:27.304+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:06:27.304] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:06:27.312+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:06:27.312] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:06:27.313+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:06:27.313] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:06:27.313+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:06:27.313] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:06:27.314+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:06:27.314] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:06:27.315+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:06:27.315] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:06:27.315+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:06:27.315] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:06:27.316+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:06:27.316] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:06:27.327+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:27.327] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:27.548+0000] [.info] [src/director/director.go:tracef:794] [01:06:27.548] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:06:27.548+0000] [.info] [src/director/director.go:tracef:794] [01:06:27.548] because ['currentState.Up' = false] [2021-10-22T01:06:27.746+0000] [.info] [src/director/director.go:tracef:794] [01:06:27.746] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:06:27.746+0000] [.info] [src/director/director.go:tracef:794] [01:06:27.746] because ['currentState.Up' = false] [2021-10-22T01:06:27.746+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:06:27.746] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:06:27.963+0000] [.info] [src/director/director.go:tracef:794] [01:06:27.963] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:06:27.964+0000] [.info] [src/director/director.go:tracef:794] [01:06:27.964] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:06:28.083+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:06:28.083] clusterConfig unchanged [2021-10-22T01:06:28.185+0000] [.info] [src/director/director.go:tracef:794] [01:06:28.185] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:06:28.186+0000] [.info] [src/director/director.go:tracef:794] [01:06:28.186] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:06:28.186+0000] [.info] [src/director/director.go:tracef:794] [01:06:28.186] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:06:28.186+0000] [.info] [src/director/director.go:tracef:794] [01:06:28.186] because ['currentState.Up' = false] [2021-10-22T01:06:28.186+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:06:28.186] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:06:28.186+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:06:28.186] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:06:28.221+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:06:28.221] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:06:28.222+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:06:28.222] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45129 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:06:28.328+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:28.328] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:28.931+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:06:28.931] ...Finished start [pid=12877] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45129,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:06:29.082+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:06:29.082] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 149.82514ms. [2021-10-22T01:06:29.082+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:06:29.082] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45129 (local=false) [2021-10-22T01:06:29.082+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:06:29.082] Dropping existing documents from system.replset [2021-10-22T01:06:29.191+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:06:29.191] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:06:29.195+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:06:29.195] Inserting oplogTruncateAfterPoint document [2021-10-22T01:06:29.195+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:06:29.195] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:06:29.196+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:06:29.196] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:06:29.196+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:06:29.196] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:06:29.208+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:06:29.208] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:06:29.208+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:06:29.208] Setting the minvalid document to [{_id ObjectID("61720e95523370002becc45c")} {t -1} {ts {0 1}}] [2021-10-22T01:06:29.215+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:06:29.215] clusterConfig unchanged [2021-10-22T01:06:29.220+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:06:29.220] Inserted [{_id ObjectID("61720e95523370002becc45c")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:06:29.220+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:06:29.220] Inserting new replica set configuration document. [2021-10-22T01:06:29.220+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:06:29.220] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:06:29.231+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:06:29.231] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:06:29.235+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:06:29.235] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:06:29.235+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:06:29.235] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:06:29.235+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:06:29.235] Initiating stopEphemeralMongo [2021-10-22T01:06:29.235+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:06:29.235] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45129 (local=false) [2021-10-22T01:06:29.235+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:06:29.235] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45129 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:06:29.330+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:29.330] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:29.662+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:06:29.662] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.83528ms. [2021-10-22T01:06:29.662+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:06:29.662] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45129 (local=false) [2021-10-22T01:06:29.662+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:06:29.662] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.332µs. [2021-10-22T01:06:29.662+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:06:29.662] Successfully finished running stopEphemeralMongo [2021-10-22T01:06:29.662+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:06:29.662] Successfully stopped existing ephemeral mongod. [2021-10-22T01:06:29.662+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:06:29.662] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:06:29.662+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:06:29.662] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:06:29.694+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:06:29.694] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:06:29.695+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:06:29.695] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39364 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:06:30.329+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:30.329] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:30.347+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:06:30.347] clusterConfig unchanged [2021-10-22T01:06:30.910+0000] [.error] [src/action/start.go:func1:93] [103] [01:06:30.910] Error starting mongod : [01:06:30.910] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39364,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39364 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:06:29.729Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 14373 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:06:29.750+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:06:29.750+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:06:29.750+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39364,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:06:29.751+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:06:29.753+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:06:29.753+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:06:29.806+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:06:29.806+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:06:29.808+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:06:29.860+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:06:29.860+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:06:30.363+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864790:363784][14373:0x7f8fbcce0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:06:30.479+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864790:479290][14373:0x7f8fbcce0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:06:30.579+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864790:579883][14373:0x7f8fbcce0bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:06:30.692+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864790:692192][14373:0x7f8fbcce0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:06:30.768+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864790:768778][14373:0x7f8fbcce0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:06:30.849+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864790:849766][14373:0x7f8fbcce0bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:06:30.849+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864790:849838][14373:0x7f8fbcce0bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:06:30.911+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:06:30.911] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39364 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:06:30.910] Error starting mongod : [01:06:30.910] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39364,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39364 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:06:29.729Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 14373 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:06:29.750+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:06:29.750+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:06:29.750+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39364,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:06:29.751+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:06:29.753+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:06:29.753+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:06:29.806+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:06:29.806+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:06:29.808+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:06:29.860+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:06:29.860+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:06:30.363+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864790:363784][14373:0x7f8fbcce0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:06:30.479+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:06:30.911+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:06:30.911] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:06:30.911] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39364 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:06:30.910] Error starting mongod : [01:06:30.910] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39364,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39364 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:06:29.729Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 14373 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:06:29.750+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:06:29.750+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:06:29.750+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39364,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:06:29.751+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:06:29.753+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:06:29.753+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:06:29.806+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:06:29.806+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:06:29.808+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:06:29.860+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:06:29.860+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:06:30.363+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864790:363784][14373:0x7f8fbcce0bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:06:30.911+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:06:30.911] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:06:30.911+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:06:30.911] Initiating stopEphemeralMongo [2021-10-22T01:06:30.911+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:06:30.911] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:06:30.912+0000] [.error] [src/director/director.go:executePlan:962] [01:06:30.912] Failed to apply action. Result = : [01:06:30.911] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:06:30.911] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39364 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:06:30.910] Error starting mongod : [01:06:30.910] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39364,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39364 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:06:29.729Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 14373 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:06:29.750+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:06:29.750+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:06:29.750+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39364,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:06:29.751+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:06:29.753+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:06:29.753+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:06:29.806+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:06:29.806+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:06:29.808+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:06:29.860+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:06:29.860+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:06:30.363+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:06:30.912+0000] [.error] [src/director/director.go:planAndExecute:567] [01:06:30.912] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:06:30.912] Failed to apply action. Result = : [01:06:30.911] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:06:30.911] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39364 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:06:30.910] Error starting mongod : [01:06:30.910] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39364,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39364 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:06:29.729Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 14373 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:06:29.750+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:06:29.750+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:06:29.750+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39364,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:06:29.751+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:06:29.753+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:06:29.753+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:06:29.806+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:06:29.806+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:06:29.808+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:06:29.860+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:06:29.860+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:06:30.912+0000] [.error] [src/director/director.go:mainLoop:399] [01:06:30.912] Failed to planAndExecute : [01:06:30.912] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:06:30.912] Failed to apply action. Result = : [01:06:30.911] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:06:30.911] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39364 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:06:30.910] Error starting mongod : [01:06:30.910] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39364,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39364 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:06:29.729Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 14373 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:06:29.750+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:06:29.750+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:06:29.750+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39364,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:06:29.751+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:06:29.753+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:06:29.753+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:06:29.806+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:06:29.806+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:06:29.808+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:06:29.860+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:06:29.860+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:06:31.328+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:31.328] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:31.587+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:06:31.587] clusterConfig unchanged [2021-10-22T01:06:32.331+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:32.331] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:33.330+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:33.330] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:34.331+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:34.331] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:35.332+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:35.332] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:36.332+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:36.331] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:37.332+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:37.332] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:38.332+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:38.332] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:39.335+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:39.335] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:39.668+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:06:39.668] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.352023945s [2021-10-22T01:06:40.333+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:40.333] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:41.293+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:06:41.293] memory:rss=51445760 alloc=8491992 totalalloc=3985081440 mallocs=51122236 frees=51061332 heapinuse=14598144 heapobjects=60904 gcpausetotalns=1447667156 gcpauselastns=141947 gcnum=635 utimens=71722643000 stimens=23700761000 minflt=196252 majflt=405 [2021-10-22T01:06:41.333+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:41.333] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:41.700+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:06:41.700] clusterConfig unchanged [2021-10-22T01:06:42.334+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:42.334] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:43.337+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:43.337] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:44.336+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:44.336] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:45.336+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:45.336] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:46.337+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:46.337] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:47.337+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:47.337] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:48.344+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:48.344] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:49.338+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:49.338] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:50.339+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:50.339] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:51.339+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:51.339] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:51.842+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:06:51.842] clusterConfig unchanged [2021-10-22T01:06:52.339+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:52.339] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:53.340+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:53.340] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:54.347+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:54.347] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:55.342+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:55.342] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:56.343+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:56.343] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:57.343+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:57.343] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:58.343+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:58.343] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:06:59.349+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:06:59.349] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:00.344+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:00.344] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:01.345+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:01.345] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:01.944+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:07:01.944] clusterConfig unchanged [2021-10-22T01:07:02.366+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:02.366] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:03.346+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:03.346] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:04.347+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:04.346] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:05.346+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:05.346] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:06.347+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:06.347] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:07.348+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:07.348] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:08.349+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:08.349] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:09.349+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:09.349] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:10.355+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:10.355] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:11.350+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:11.350] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:12.046+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:07:12.046] clusterConfig unchanged [2021-10-22T01:07:12.351+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:12.351] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:13.351+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:13.351] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:14.352+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:14.352] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:15.353+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:15.353] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:16.353+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:16.353] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:17.354+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:17.354] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:18.355+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:18.355] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:19.356+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:19.356] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:20.357+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:20.357] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:21.362+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:21.362] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:22.187+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:07:22.187] clusterConfig unchanged [2021-10-22T01:07:22.359+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:22.359] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:23.361+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:23.361] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:24.360+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:24.360] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:25.360+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:25.360] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:26.361+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:26.361] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:27.362+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:27.362] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:27.559+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:07:27.559] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:07:27.561+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:07:27.561] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:07:28.363+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:28.363] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:29.363+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:29.363] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:30.364+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:30.364] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:31.076+0000] [.info] [src/director/director.go:computePlan:280] [01:07:31.076] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:07:31.077+0000] [.info] [src/director/director.go:tracef:794] [01:07:31.077] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:07:31.077+0000] [.info] [src/director/director.go:tracef:794] [01:07:31.077] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:07:31.077+0000] [.info] [src/director/director.go:tracef:794] [01:07:31.077] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:07:31.077+0000] [.info] [src/director/director.go:tracef:794] [01:07:31.077] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:07:31.077+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:07:31.077] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:07:31.228+0000] [.info] [src/director/director.go:tracef:794] [01:07:31.228] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:07:31.228+0000] [.info] [src/director/director.go:tracef:794] [01:07:31.228] because [2021-10-22T01:07:31.229+0000] [.info] [src/director/director.go:tracef:794] [01:07:31.229] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:07:31.229+0000] [.info] [src/director/director.go:tracef:794] [01:07:31.229] because ['currentState.CanBounceStop' = true] [2021-10-22T01:07:31.366+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:31.366] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:31.387+0000] [.info] [src/director/director.go:tracef:794] [01:07:31.387] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:07:31.387+0000] [.info] [src/director/director.go:tracef:794] [01:07:31.387] because [2021-10-22T01:07:31.388+0000] [.info] [src/director/director.go:tracef:794] [01:07:31.388] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:07:31.388+0000] [.info] [src/director/director.go:tracef:794] [01:07:31.388] because ['currentState.Up' = false] [2021-10-22T01:07:31.388+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:07:31.388] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:07:31.388+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:07:31.388] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:07:31.472+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:07:31.472] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:07:31.480+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:07:31.480] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:07:31.481+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:07:31.481] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:07:31.481+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:07:31.481] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:07:31.483+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:07:31.483] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:07:31.483+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:07:31.483] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:07:31.483+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:07:31.483] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:07:31.485+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:07:31.485] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:07:31.699+0000] [.info] [src/director/director.go:tracef:794] [01:07:31.699] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:07:31.699+0000] [.info] [src/director/director.go:tracef:794] [01:07:31.699] because ['currentState.Up' = false] [2021-10-22T01:07:31.900+0000] [.info] [src/director/director.go:tracef:794] [01:07:31.900] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:07:31.900+0000] [.info] [src/director/director.go:tracef:794] [01:07:31.900] because ['currentState.Up' = false] [2021-10-22T01:07:31.900+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:07:31.900] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:07:32.104+0000] [.info] [src/director/director.go:tracef:794] [01:07:32.104] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:07:32.104+0000] [.info] [src/director/director.go:tracef:794] [01:07:32.104] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:07:32.307+0000] [.info] [src/director/director.go:tracef:794] [01:07:32.307] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:07:32.307+0000] [.info] [src/director/director.go:tracef:794] [01:07:32.307] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:07:32.307+0000] [.info] [src/director/director.go:tracef:794] [01:07:32.307] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:07:32.307+0000] [.info] [src/director/director.go:tracef:794] [01:07:32.307] because ['currentState.Up' = false] [2021-10-22T01:07:32.307+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:07:32.307] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:07:32.307+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:07:32.307] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:07:32.340+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:07:32.340] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:07:32.340+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:07:32.340] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46414 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:07:32.366+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:32.366] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:32.398+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:07:32.398] clusterConfig unchanged [2021-10-22T01:07:33.051+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:07:33.051] ...Finished start [pid=33449] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46414,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:07:33.173+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:07:33.173] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 121.697123ms. [2021-10-22T01:07:33.173+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:07:33.173] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46414 (local=false) [2021-10-22T01:07:33.173+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:07:33.173] Dropping existing documents from system.replset [2021-10-22T01:07:33.311+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:07:33.311] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:07:33.314+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:07:33.314] Inserting oplogTruncateAfterPoint document [2021-10-22T01:07:33.314+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:07:33.314] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:07:33.315+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:07:33.315] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:07:33.315+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:07:33.315] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:07:33.327+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:07:33.327] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:07:33.327+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:07:33.327] Setting the minvalid document to [{_id ObjectID("61720ed5523370002becc46a")} {t -1} {ts {0 1}}] [2021-10-22T01:07:33.339+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:07:33.339] Inserted [{_id ObjectID("61720ed5523370002becc46a")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:07:33.339+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:07:33.339] Inserting new replica set configuration document. [2021-10-22T01:07:33.339+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:07:33.339] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:07:33.351+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:07:33.351] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:07:33.354+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:07:33.354] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:07:33.354+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:07:33.354] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:07:33.355+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:07:33.354] Initiating stopEphemeralMongo [2021-10-22T01:07:33.355+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:07:33.355] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46414 (local=false) [2021-10-22T01:07:33.355+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:07:33.355] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46414 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:07:33.380+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:33.380] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:33.510+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:07:33.510] clusterConfig unchanged [2021-10-22T01:07:33.711+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:07:33.711] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.042069ms. [2021-10-22T01:07:33.711+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:07:33.711] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46414 (local=false) [2021-10-22T01:07:33.712+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:07:33.712] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.875µs. [2021-10-22T01:07:33.712+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:07:33.712] Successfully finished running stopEphemeralMongo [2021-10-22T01:07:33.712+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:07:33.712] Successfully stopped existing ephemeral mongod. [2021-10-22T01:07:33.712+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:07:33.712] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:07:33.712+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:07:33.712] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:07:33.748+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:07:33.748] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:07:33.749+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:07:33.749] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33980 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:07:34.368+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:34.368] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:34.642+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:07:34.642] clusterConfig unchanged [2021-10-22T01:07:34.918+0000] [.error] [src/action/start.go:func1:93] [103] [01:07:34.918] Error starting mongod : [01:07:34.917] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33980,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33980 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 35079 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:07:33.812+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:07:33.812+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:07:33.812+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33980,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:07:33.814+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:07:33.818+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:07:33.818+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:07:33.830+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:07:33.830+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:07:33.849+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:07:33.894+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:07:33.894+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:07:34.419+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864854:419969][35079:0x7f04c1105bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:07:34.522+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864854:522606][35079:0x7f04c1105bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:07:34.619+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864854:619485][35079:0x7f04c1105bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:07:34.729+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864854:729539][35079:0x7f04c1105bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:07:34.811+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864854:811166][35079:0x7f04c1105bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:07:34.866+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864854:866950][35079:0x7f04c1105bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:07:34.867+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864854:867040][35079:0x7f04c1105bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:07:34.868+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:07:34.918+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:07:34.918] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33980 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:07:34.918] Error starting mongod : [01:07:34.917] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33980,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33980 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 35079 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:07:33.812+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:07:33.812+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:07:33.812+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33980,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:07:33.814+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:07:33.818+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:07:33.818+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:07:33.830+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:07:33.830+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:07:33.849+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:07:33.894+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:07:33.894+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:07:34.419+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864854:419969][35079:0x7f04c1105bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:07:34.522+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864854:522606][35079:0x7f04c1105bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:07:34.918+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:07:34.918] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:07:34.918] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33980 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:07:34.918] Error starting mongod : [01:07:34.917] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33980,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33980 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 35079 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:07:33.812+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:07:33.812+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:07:33.812+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33980,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:07:33.814+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:07:33.818+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:07:33.818+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:07:33.830+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:07:33.830+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:07:33.849+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:07:33.894+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:07:33.894+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:07:34.419+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864854:419969][35079:0x7f04c1105bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:07:34.522+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:07:34.918+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:07:34.918] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:07:34.918+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:07:34.918] Initiating stopEphemeralMongo [2021-10-22T01:07:34.918+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:07:34.918] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:07:34.919+0000] [.error] [src/director/director.go:executePlan:962] [01:07:34.919] Failed to apply action. Result = : [01:07:34.918] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:07:34.918] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33980 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:07:34.918] Error starting mongod : [01:07:34.917] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33980,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33980 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 35079 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:07:33.812+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:07:33.812+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:07:33.812+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33980,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:07:33.814+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:07:33.818+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:07:33.818+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:07:33.830+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:07:33.830+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:07:33.849+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:07:33.894+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:07:33.894+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:07:34.419+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864854:419969][35079:0x7f04c1105bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:07:34.919+0000] [.error] [src/director/director.go:planAndExecute:567] [01:07:34.919] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:07:34.919] Failed to apply action. Result = : [01:07:34.918] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:07:34.918] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33980 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:07:34.918] Error starting mongod : [01:07:34.917] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33980,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33980 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 35079 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:07:33.812+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:07:33.812+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:07:33.812+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33980,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:07:33.814+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:07:33.818+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:07:33.818+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:07:33.830+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:07:33.830+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:07:33.849+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:07:33.894+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:07:33.894+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:07:34.419+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:07:34.919+0000] [.error] [src/director/director.go:mainLoop:399] [01:07:34.919] Failed to planAndExecute : [01:07:34.919] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:07:34.919] Failed to apply action. Result = : [01:07:34.918] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:07:34.918] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33980 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:07:34.918] Error starting mongod : [01:07:34.917] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33980,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33980 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 35079 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:07:33.812+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:07:33.812+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:07:33.812+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33980,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:07:33.814+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:07:33.818+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:07:33.818+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:07:33.830+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:07:33.830+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:07:33.849+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:07:33.894+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:07:33.894+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:07:34.419+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:07:35.383+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [01:07:35.383] lsblk cache invalidated, running lsblk command [2021-10-22T01:07:35.386+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:35.386] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:35.913+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:07:35.913] clusterConfig unchanged [2021-10-22T01:07:36.366+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:36.366] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:37.369+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:37.369] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:38.370+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:38.370] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:39.369+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:39.369] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:40.371+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:40.371] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:41.302+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:07:41.302] memory:rss=52555776 alloc=13262560 totalalloc=4070576856 mallocs=52211507 frees=52106427 heapinuse=16605184 heapobjects=105080 gcpausetotalns=1450760967 gcpauselastns=114794 gcnum=647 utimens=73320156000 stimens=24268746000 minflt=199944 majflt=405 [2021-10-22T01:07:41.372+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:41.372] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:42.372+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:42.372] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:43.374+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:43.374] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:43.696+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:07:43.696] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.211507159s [2021-10-22T01:07:44.371+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:44.371] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:45.372+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:45.372] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:46.034+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:07:46.034] clusterConfig unchanged [2021-10-22T01:07:46.372+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:46.372] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:47.373+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:47.373] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:48.373+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:48.373] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:49.374+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:49.374] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:50.376+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:50.376] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:51.379+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:51.378] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:52.375+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:52.375] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:53.376+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:53.376] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:54.376+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:54.376] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:55.377+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:55.377] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:56.136+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:07:56.136] clusterConfig unchanged [2021-10-22T01:07:56.378+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:56.378] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:57.378+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:57.378] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:58.379+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:58.379] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:07:59.379+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:07:59.379] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:00.382+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:00.382] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:01.381+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:01.381] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:02.382+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:02.381] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:03.381+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:03.381] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:04.382+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:04.382] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:05.382+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:05.382] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:06.278+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:08:06.278] clusterConfig unchanged [2021-10-22T01:08:06.383+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:06.383] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:07.383+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:07.383] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:08.384+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:08.384] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:09.384+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:09.384] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:10.384+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:10.384] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:11.384+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:11.384] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:12.386+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:12.386] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:13.393+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:13.392] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:14.387+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:14.387] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:15.388+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:15.388] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:16.389+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:16.389] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:16.400+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:08:16.400] clusterConfig unchanged [2021-10-22T01:08:17.389+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:17.389] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:18.390+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:18.390] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:19.391+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:19.391] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:20.393+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:20.393] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:21.398+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:21.398] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:22.393+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:22.393] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:23.393+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:23.393] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:24.394+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:24.394] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:25.395+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:25.395] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:26.396+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:26.396] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:26.523+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:08:26.523] clusterConfig unchanged [2021-10-22T01:08:27.396+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:27.396] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:28.396+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:28.396] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:29.397+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:29.397] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:30.397+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:30.397] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:31.398+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:31.398] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:32.398+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:32.398] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:33.398+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:33.398] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:34.398+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:34.398] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:35.092+0000] [.info] [src/director/director.go:computePlan:280] [01:08:35.091] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:08:35.092+0000] [.info] [src/director/director.go:tracef:794] [01:08:35.092] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:08:35.092+0000] [.info] [src/director/director.go:tracef:794] [01:08:35.092] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:08:35.092+0000] [.info] [src/director/director.go:tracef:794] [01:08:35.092] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:08:35.092+0000] [.info] [src/director/director.go:tracef:794] [01:08:35.092] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:08:35.092+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:08:35.092] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:08:35.252+0000] [.info] [src/director/director.go:tracef:794] [01:08:35.252] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:08:35.252+0000] [.info] [src/director/director.go:tracef:794] [01:08:35.252] because [2021-10-22T01:08:35.253+0000] [.info] [src/director/director.go:tracef:794] [01:08:35.252] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:08:35.253+0000] [.info] [src/director/director.go:tracef:794] [01:08:35.253] because ['currentState.CanBounceStop' = true] [2021-10-22T01:08:35.401+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:35.401] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:35.417+0000] [.info] [src/director/director.go:tracef:794] [01:08:35.417] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:08:35.417+0000] [.info] [src/director/director.go:tracef:794] [01:08:35.417] because [2021-10-22T01:08:35.417+0000] [.info] [src/director/director.go:tracef:794] [01:08:35.417] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:08:35.417+0000] [.info] [src/director/director.go:tracef:794] [01:08:35.417] because ['currentState.Up' = false] [2021-10-22T01:08:35.417+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:08:35.417] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:08:35.417+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:08:35.417] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:08:35.502+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:08:35.502] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:08:35.510+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:08:35.510] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:08:35.511+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:08:35.511] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:08:35.512+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:08:35.512] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:08:35.512+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:08:35.512] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:08:35.512+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:08:35.512] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:08:35.514+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:08:35.514] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:08:35.724+0000] [.info] [src/director/director.go:tracef:794] [01:08:35.724] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:08:35.724+0000] [.info] [src/director/director.go:tracef:794] [01:08:35.724] because ['currentState.Up' = false] [2021-10-22T01:08:35.938+0000] [.info] [src/director/director.go:tracef:794] [01:08:35.938] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:08:35.938+0000] [.info] [src/director/director.go:tracef:794] [01:08:35.938] because ['currentState.Up' = false] [2021-10-22T01:08:35.938+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:08:35.938] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:08:36.148+0000] [.info] [src/director/director.go:tracef:794] [01:08:36.148] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:08:36.148+0000] [.info] [src/director/director.go:tracef:794] [01:08:36.148] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:08:36.373+0000] [.info] [src/director/director.go:tracef:794] [01:08:36.373] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:08:36.373+0000] [.info] [src/director/director.go:tracef:794] [01:08:36.373] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:08:36.373+0000] [.info] [src/director/director.go:tracef:794] [01:08:36.373] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:08:36.373+0000] [.info] [src/director/director.go:tracef:794] [01:08:36.373] because ['currentState.Up' = false] [2021-10-22T01:08:36.373+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:08:36.373] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:08:36.373+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:08:36.373] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:08:36.414+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:08:36.414] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:08:36.415+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:08:36.414] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36484 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:08:36.416+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:36.416] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:36.716+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:08:36.716] clusterConfig unchanged [2021-10-22T01:08:37.143+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:08:37.142] ...Finished start [pid=54324] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36484,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:08:37.285+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:08:37.285] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 142.245672ms. [2021-10-22T01:08:37.285+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:08:37.285] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36484 (local=false) [2021-10-22T01:08:37.285+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:08:37.285] Dropping existing documents from system.replset [2021-10-22T01:08:37.411+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:37.411] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:37.443+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:08:37.443] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:08:37.489+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:08:37.489] Inserting oplogTruncateAfterPoint document [2021-10-22T01:08:37.489+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:08:37.489] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:08:37.490+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:08:37.490] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:08:37.490+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:08:37.490] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:08:37.502+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:08:37.502] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:08:37.502+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:08:37.502] Setting the minvalid document to [{_id ObjectID("61720f15523370002becc478")} {t -1} {ts {0 1}}] [2021-10-22T01:08:37.515+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:08:37.515] Inserted [{_id ObjectID("61720f15523370002becc478")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:08:37.515+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:08:37.515] Inserting new replica set configuration document. [2021-10-22T01:08:37.515+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:08:37.515] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:08:37.529+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:08:37.529] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:08:37.562+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:08:37.562] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:08:37.562+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:08:37.562] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:08:37.562+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:08:37.562] Initiating stopEphemeralMongo [2021-10-22T01:08:37.562+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:08:37.562] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36484 (local=false) [2021-10-22T01:08:37.562+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:08:37.562] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36484 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:08:37.845+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:08:37.845] clusterConfig unchanged [2021-10-22T01:08:37.995+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:08:37.995] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.028588ms. [2021-10-22T01:08:37.995+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:08:37.995] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36484 (local=false) [2021-10-22T01:08:37.995+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:08:37.995] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 22.668µs. [2021-10-22T01:08:37.995+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:08:37.995] Successfully finished running stopEphemeralMongo [2021-10-22T01:08:37.995+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:08:37.995] Successfully stopped existing ephemeral mongod. [2021-10-22T01:08:37.995+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:08:37.995] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:08:37.995+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:08:37.995] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:08:38.031+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:08:38.031] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:08:38.031+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:08:38.031] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32885 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:08:38.413+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:38.413] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:38.974+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:08:38.974] clusterConfig unchanged [2021-10-22T01:08:39.209+0000] [.error] [src/action/start.go:func1:93] [103] [01:08:39.209] Error starting mongod : [01:08:39.209] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32885 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 55885 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:08:38.082+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:08:38.082+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:08:38.082+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:08:38.084+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:08:38.086+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:08:38.086+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:08:38.123+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:08:38.123+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:08:38.123+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:08:38.180+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:08:38.180+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:08:38.708+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864918:708024][55885:0x7f51f2191bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:08:38.810+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864918:810897][55885:0x7f51f2191bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:08:38.910+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864918:910685][55885:0x7f51f2191bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/33920 to 2/256"}} {"t":{"$date":"2021-10-22T01:08:39.021+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864919:21843][55885:0x7f51f2191bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:08:39.093+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864919:93262][55885:0x7f51f2191bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:08:39.149+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864919:149579][55885:0x7f51f2191bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:08:39.149+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864919:149652][55885:0x7f51f2191bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:08:39.151+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:08:39.209+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:08:39.209] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:32885 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:08:39.209] Error starting mongod : [01:08:39.209] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32885 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 55885 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:08:38.082+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:08:38.082+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:08:38.082+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:08:38.084+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:08:38.086+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:08:38.086+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:08:38.123+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:08:38.123+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:08:38.123+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:08:38.180+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:08:38.180+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:08:38.708+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864918:708024][55885:0x7f51f2191bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:08:38.810+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864918:810897][55885:0x7f51f2191bc0], txn-recover: [...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:08:39.210+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:08:39.209] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:08:39.209] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:32885 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:08:39.209] Error starting mongod : [01:08:39.209] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32885 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 55885 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:08:38.082+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:08:38.082+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:08:38.082+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:08:38.084+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:08:38.086+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:08:38.086+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:08:38.123+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:08:38.123+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:08:38.123+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:08:38.180+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:08:38.180+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:08:38.708+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864918:708024][55885:0x7f51f2191bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:08:38.810+00:00"},"s":"I", "c":"STORAGE", "id":22430, "c... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:08:39.210+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:08:39.210] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:08:39.210+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:08:39.210] Initiating stopEphemeralMongo [2021-10-22T01:08:39.210+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:08:39.210] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:08:39.210+0000] [.error] [src/director/director.go:executePlan:962] [01:08:39.210] Failed to apply action. Result = : [01:08:39.209] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:08:39.209] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:32885 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:08:39.209] Error starting mongod : [01:08:39.209] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32885 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 55885 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:08:38.082+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:08:38.082+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:08:38.082+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:08:38.084+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:08:38.086+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:08:38.086+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:08:38.123+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:08:38.123+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:08:38.123+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:08:38.180+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:08:38.180+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:08:38.708+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864918:708024][55885:0x7f51f2191bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:08:39.210+0000] [.error] [src/director/director.go:planAndExecute:567] [01:08:39.210] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:08:39.210] Failed to apply action. Result = : [01:08:39.209] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:08:39.209] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:32885 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:08:39.209] Error starting mongod : [01:08:39.209] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32885 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 55885 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:08:38.082+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:08:38.082+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:08:38.082+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:08:38.084+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:08:38.086+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:08:38.086+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:08:38.123+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:08:38.123+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:08:38.123+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:08:38.180+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:08:38.180+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:08:38.708+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger messag... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:08:39.210+0000] [.error] [src/director/director.go:mainLoop:399] [01:08:39.210] Failed to planAndExecute : [01:08:39.210] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:08:39.210] Failed to apply action. Result = : [01:08:39.209] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:08:39.209] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:32885 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:08:39.209] Error starting mongod : [01:08:39.209] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32885 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 55885 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:08:38.082+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:08:38.082+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:08:38.082+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:08:38.084+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:08:38.086+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:08:38.086+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:08:38.123+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:08:38.123+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:08:38.123+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:08:38.180+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:08:38.180+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:08:38.708+00:00"},"s":"I", "c"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:08:39.412+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:39.412] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:40.247+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:08:40.247] clusterConfig unchanged [2021-10-22T01:08:40.412+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:40.412] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:41.302+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:08:41.302] memory:rss=53293056 alloc=13461824 totalalloc=4155651952 mallocs=53295521 frees=53202143 heapinuse=17965056 heapobjects=93378 gcpausetotalns=1451994294 gcpauselastns=91416 gcnum=660 utimens=75167472000 stimens=24965411000 minflt=204745 majflt=405 [2021-10-22T01:08:41.411+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:41.411] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:42.412+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:42.412] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:43.414+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:43.414] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:44.210+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [01:08:44.210] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:32885 (local=false) because it is not up [2021-10-22T01:08:44.414+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:44.414] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:45.414+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:45.414] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:46.415+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:46.415] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:47.416+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:47.416] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:47.953+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:08:47.953] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.43948594s [2021-10-22T01:08:48.414+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:48.414] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:49.415+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:49.415] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:50.399+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:08:50.399] clusterConfig unchanged [2021-10-22T01:08:50.416+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:50.416] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:51.417+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:51.417] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:52.417+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:52.417] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:53.418+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:53.418] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:54.418+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:54.418] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:55.418+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:55.418] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:56.419+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:56.419] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:57.420+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:57.419] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:58.420+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:58.420] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:08:59.420+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:08:59.420] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:00.422+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:00.422] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:00.530+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:09:00.530] clusterConfig unchanged [2021-10-22T01:09:01.425+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:01.425] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:02.422+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:02.422] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:03.423+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:03.423] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:04.423+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:04.423] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:05.423+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:05.423] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:06.424+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:06.424] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:07.423+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:07.423] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:08.425+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:08.425] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:09.425+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:09.425] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:10.426+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:10.426] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:10.672+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:09:10.672] clusterConfig unchanged [2021-10-22T01:09:11.431+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:11.431] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:12.429+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:12.429] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:13.428+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:13.428] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:14.428+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:14.428] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:15.428+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:15.428] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:16.429+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:16.429] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:17.430+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:17.430] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:18.431+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:18.431] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:19.431+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:19.431] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:20.431+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:20.431] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:20.804+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:09:20.804] clusterConfig unchanged [2021-10-22T01:09:21.433+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:21.433] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:22.433+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:22.433] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:23.440+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:23.439] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:24.436+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:24.436] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:25.435+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:25.435] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:26.435+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:26.435] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:27.436+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:27.436] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:28.452+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:28.451] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:29.437+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:29.437] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:30.438+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:30.438] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:30.936+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:09:30.936] clusterConfig unchanged [2021-10-22T01:09:31.438+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:31.438] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:32.439+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:32.439] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:33.439+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:33.439] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:34.455+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:34.455] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:35.441+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:35.441] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:36.443+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:36.443] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:37.442+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:37.442] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:38.442+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:38.442] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:39.374+0000] [.info] [src/director/director.go:computePlan:280] [01:09:39.374] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:09:39.374+0000] [.info] [src/director/director.go:tracef:794] [01:09:39.374] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:09:39.374+0000] [.info] [src/director/director.go:tracef:794] [01:09:39.374] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:09:39.374+0000] [.info] [src/director/director.go:tracef:794] [01:09:39.374] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:09:39.374+0000] [.info] [src/director/director.go:tracef:794] [01:09:39.374] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:09:39.375+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:09:39.375] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:09:39.443+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:39.443] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:39.534+0000] [.info] [src/director/director.go:tracef:794] [01:09:39.534] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:09:39.534+0000] [.info] [src/director/director.go:tracef:794] [01:09:39.534] because [2021-10-22T01:09:39.534+0000] [.info] [src/director/director.go:tracef:794] [01:09:39.534] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:09:39.534+0000] [.info] [src/director/director.go:tracef:794] [01:09:39.534] because ['currentState.CanBounceStop' = true] [2021-10-22T01:09:39.687+0000] [.info] [src/director/director.go:tracef:794] [01:09:39.687] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:09:39.687+0000] [.info] [src/director/director.go:tracef:794] [01:09:39.687] because [2021-10-22T01:09:39.687+0000] [.info] [src/director/director.go:tracef:794] [01:09:39.687] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:09:39.687+0000] [.info] [src/director/director.go:tracef:794] [01:09:39.687] because ['currentState.Up' = false] [2021-10-22T01:09:39.687+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:09:39.687] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:09:39.687+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:09:39.687] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:09:39.768+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:09:39.768] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:09:39.776+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:09:39.776] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:09:39.777+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:09:39.777] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:09:39.778+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:09:39.778] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:09:39.778+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:09:39.778] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:09:39.778+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:09:39.778] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:09:39.780+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:09:39.780] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:09:39.974+0000] [.info] [src/director/director.go:tracef:794] [01:09:39.974] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:09:39.974+0000] [.info] [src/director/director.go:tracef:794] [01:09:39.974] because ['currentState.Up' = false] [2021-10-22T01:09:40.184+0000] [.info] [src/director/director.go:tracef:794] [01:09:40.184] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:09:40.184+0000] [.info] [src/director/director.go:tracef:794] [01:09:40.184] because ['currentState.Up' = false] [2021-10-22T01:09:40.184+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:09:40.184] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:09:40.416+0000] [.info] [src/director/director.go:tracef:794] [01:09:40.416] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:09:40.416+0000] [.info] [src/director/director.go:tracef:794] [01:09:40.416] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:09:40.456+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:40.456] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:40.621+0000] [.info] [src/director/director.go:tracef:794] [01:09:40.621] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:09:40.622+0000] [.info] [src/director/director.go:tracef:794] [01:09:40.622] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:09:40.622+0000] [.info] [src/director/director.go:tracef:794] [01:09:40.622] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:09:40.622+0000] [.info] [src/director/director.go:tracef:794] [01:09:40.622] because ['currentState.Up' = false] [2021-10-22T01:09:40.622+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:09:40.622] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:09:40.622+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:09:40.622] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:09:40.655+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:09:40.655] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:09:40.655+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:09:40.655] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 44592 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:09:41.176+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:09:41.176] clusterConfig unchanged [2021-10-22T01:09:41.310+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:09:41.310] memory:rss=53252096 alloc=15019256 totalalloc=4229931840 mallocs=54268197 frees=54125866 heapinuse=17924096 heapobjects=142331 gcpausetotalns=1453163151 gcpauselastns=121451 gcnum=671 utimens=76523217000 stimens=25504248000 minflt=209018 majflt=405 [2021-10-22T01:09:41.390+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:09:41.390] ...Finished start [pid=949] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44592,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:09:41.453+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:41.453] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:41.551+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:09:41.551] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 161.173519ms. [2021-10-22T01:09:41.551+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:09:41.551] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44592 (local=false) [2021-10-22T01:09:41.551+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:09:41.551] Dropping existing documents from system.replset [2021-10-22T01:09:41.748+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:09:41.748] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:09:41.751+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:09:41.751] Inserting oplogTruncateAfterPoint document [2021-10-22T01:09:41.751+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:09:41.751] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:09:41.752+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:09:41.752] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:09:41.752+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:09:41.752] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:09:41.765+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:09:41.765] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:09:41.765+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:09:41.765] Setting the minvalid document to [{_id ObjectID("61720f55523370002becc486")} {t -1} {ts {0 1}}] [2021-10-22T01:09:41.778+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:09:41.777] Inserted [{_id ObjectID("61720f55523370002becc486")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:09:41.778+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:09:41.778] Inserting new replica set configuration document. [2021-10-22T01:09:41.778+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:09:41.778] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:09:41.790+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:09:41.790] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:09:41.823+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:09:41.823] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:09:41.823+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:09:41.823] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:09:41.823+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:09:41.823] Initiating stopEphemeralMongo [2021-10-22T01:09:41.823+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:09:41.823] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44592 (local=false) [2021-10-22T01:09:41.823+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:09:41.823] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44592 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:09:42.262+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:09:42.262] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.094914ms. [2021-10-22T01:09:42.262+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:09:42.262] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44592 (local=false) [2021-10-22T01:09:42.262+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:09:42.262] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.904µs. [2021-10-22T01:09:42.262+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:09:42.262] Successfully finished running stopEphemeralMongo [2021-10-22T01:09:42.262+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:09:42.262] Successfully stopped existing ephemeral mongod. [2021-10-22T01:09:42.262+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:09:42.262] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:09:42.262+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:09:42.262] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:09:42.294+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:09:42.294] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:09:42.295+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:09:42.295] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53618 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:09:42.307+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:09:42.307] clusterConfig unchanged [2021-10-22T01:09:42.454+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:42.454] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:43.400+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:09:43.400] clusterConfig unchanged [2021-10-22T01:09:43.455+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:43.455] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:43.500+0000] [.error] [src/action/start.go:func1:93] [103] [01:09:43.500] Error starting mongod : [01:09:43.500] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53618,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53618 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 2592 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:09:42.350+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:09:42.350+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:09:42.350+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53618,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:09:42.352+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:09:42.354+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:09:42.360+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:09:42.366+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:09:42.366+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:09:42.367+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:09:42.412+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:09:42.412+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:09:42.974+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864982:974816][2592:0x7f4bfda8cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:09:43.099+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864983:98995][2592:0x7f4bfda8cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:09:43.200+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864983:200646][2592:0x7f4bfda8cbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:09:43.314+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864983:314163][2592:0x7f4bfda8cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:09:43.385+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864983:385660][2592:0x7f4bfda8cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:09:43.449+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864983:449500][2592:0x7f4bfda8cbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:09:43.449+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864983:449574][2592:0x7f4bfda8cbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:09:43.453+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:09:43.500+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:09:43.500] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53618 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:09:43.500] Error starting mongod : [01:09:43.500] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53618,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53618 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 2592 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:09:42.350+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:09:42.350+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:09:42.350+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53618,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:09:42.352+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:09:42.354+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:09:42.360+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:09:42.366+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:09:42.366+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:09:42.367+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:09:42.412+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:09:42.412+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:09:42.974+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864982:974816][2592:0x7f4bfda8cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:09:43.099+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864983:98995][2592:0x7f4bfda8cbc0], txn-recover...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:09:43.500+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:09:43.500] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:09:43.500] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53618 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:09:43.500] Error starting mongod : [01:09:43.500] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53618,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53618 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 2592 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:09:42.350+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:09:42.350+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:09:42.350+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53618,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:09:42.352+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:09:42.354+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:09:42.360+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:09:42.366+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:09:42.366+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:09:42.367+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:09:42.412+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:09:42.412+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:09:42.974+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864982:974816][2592:0x7f4bfda8cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:09:43.099+00:00"},"s":"I", "c":"STORAGE", "id":22430,... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:09:43.501+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:09:43.501] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:09:43.501+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:09:43.501] Initiating stopEphemeralMongo [2021-10-22T01:09:43.501+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:09:43.501] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:09:43.501+0000] [.error] [src/director/director.go:executePlan:962] [01:09:43.501] Failed to apply action. Result = : [01:09:43.500] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:09:43.500] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53618 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:09:43.500] Error starting mongod : [01:09:43.500] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53618,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53618 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 2592 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:09:42.350+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:09:42.350+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:09:42.350+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53618,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:09:42.352+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:09:42.354+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:09:42.360+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:09:42.366+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:09:42.366+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:09:42.367+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:09:42.412+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:09:42.412+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:09:42.974+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634864982:974816][2592:0x7f4bfda8cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:09:43.501+0000] [.error] [src/director/director.go:planAndExecute:567] [01:09:43.501] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:09:43.501] Failed to apply action. Result = : [01:09:43.500] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:09:43.500] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53618 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:09:43.500] Error starting mongod : [01:09:43.500] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53618,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53618 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 2592 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:09:42.350+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:09:42.350+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:09:42.350+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53618,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:09:42.352+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:09:42.354+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:09:42.360+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:09:42.366+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:09:42.366+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:09:42.367+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:09:42.412+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:09:42.412+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:09:42.974+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger ... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:09:43.501+0000] [.error] [src/director/director.go:mainLoop:399] [01:09:43.501] Failed to planAndExecute : [01:09:43.501] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:09:43.501] Failed to apply action. Result = : [01:09:43.500] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:09:43.500] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53618 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:09:43.500] Error starting mongod : [01:09:43.500] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53618,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53618 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 2592 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:09:42.350+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:09:42.350+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:09:42.350+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53618,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:09:42.352+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:09:42.354+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:09:42.360+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:09:42.366+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:09:42.366+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:09:42.367+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:09:42.412+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:09:42.412+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:09:42.974+00:00"},"s":"I"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:09:44.456+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:44.456] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:44.652+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:09:44.652] clusterConfig unchanged [2021-10-22T01:09:45.463+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:45.463] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:46.457+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:46.457] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:47.455+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:47.455] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:48.458+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:48.458] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:49.469+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:49.469] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:50.470+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:50.470] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:51.468+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:51.468] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:52.412+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:09:52.412] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.632527068s [2021-10-22T01:09:52.468+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:52.468] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:53.468+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:53.468] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:54.469+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:54.468] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:54.773+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:09:54.773] clusterConfig unchanged [2021-10-22T01:09:55.469+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:55.469] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:56.470+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:56.470] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:57.469+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:57.469] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:58.470+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:58.470] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:09:59.471+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:09:59.471] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:00.472+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:00.472] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:01.474+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:01.474] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:02.473+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:02.473] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:03.479+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:03.479] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:04.475+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:04.475] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:04.914+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:10:04.914] clusterConfig unchanged [2021-10-22T01:10:05.476+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:05.476] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:06.477+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:06.477] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:07.478+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:07.478] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:08.479+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:08.479] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:09.480+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:09.479] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:10.480+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:10.480] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:11.483+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:11.483] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:12.481+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:12.481] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:13.482+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:13.482] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:14.482+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:14.482] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:15.016+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:10:15.016] clusterConfig unchanged [2021-10-22T01:10:15.488+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:15.488] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:16.483+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:16.483] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:17.484+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:17.484] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:18.489+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:18.489] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:19.485+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:19.485] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:20.485+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:20.485] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:21.485+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:21.485] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:22.485+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:22.485] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:23.486+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:23.486] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:24.487+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:24.487] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:25.158+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:10:25.158] clusterConfig unchanged [2021-10-22T01:10:25.488+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:25.488] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:26.492+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:26.492] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:27.490+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:27.490] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:28.493+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:28.493] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:29.491+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:29.491] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:30.491+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:30.491] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:31.492+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:31.492] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:32.493+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:32.493] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:33.493+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:33.493] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:34.494+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:34.494] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:35.250+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:10:35.250] clusterConfig unchanged [2021-10-22T01:10:35.494+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:35.494] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:36.494+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:36.494] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:37.495+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:37.495] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:38.499+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:38.499] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:39.497+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:39.497] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:40.498+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:40.498] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:41.311+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:10:41.311] memory:rss=52015104 alloc=8228752 totalalloc=4275684136 mallocs=54789135 frees=54730058 heapinuse=14196736 heapobjects=59077 gcpausetotalns=1454228761 gcpauselastns=69147 gcnum=679 utimens=77886106000 stimens=26116213000 minflt=212187 majflt=405 [2021-10-22T01:10:41.499+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:41.499] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:42.500+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:42.500] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:43.501+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:43.501] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:43.670+0000] [.info] [src/director/director.go:computePlan:280] [01:10:43.670] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:10:43.670+0000] [.info] [src/director/director.go:tracef:794] [01:10:43.670] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:10:43.670+0000] [.info] [src/director/director.go:tracef:794] [01:10:43.670] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:10:43.670+0000] [.info] [src/director/director.go:tracef:794] [01:10:43.670] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:10:43.670+0000] [.info] [src/director/director.go:tracef:794] [01:10:43.670] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:10:43.670+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:10:43.670] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:10:43.827+0000] [.info] [src/director/director.go:tracef:794] [01:10:43.827] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:10:43.827+0000] [.info] [src/director/director.go:tracef:794] [01:10:43.827] because [2021-10-22T01:10:43.827+0000] [.info] [src/director/director.go:tracef:794] [01:10:43.827] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:10:43.827+0000] [.info] [src/director/director.go:tracef:794] [01:10:43.827] because ['currentState.CanBounceStop' = true] [2021-10-22T01:10:43.978+0000] [.info] [src/director/director.go:tracef:794] [01:10:43.978] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:10:43.978+0000] [.info] [src/director/director.go:tracef:794] [01:10:43.978] because [2021-10-22T01:10:43.979+0000] [.info] [src/director/director.go:tracef:794] [01:10:43.979] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:10:43.979+0000] [.info] [src/director/director.go:tracef:794] [01:10:43.979] because ['currentState.Up' = false] [2021-10-22T01:10:43.979+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:10:43.979] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:10:43.979+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:10:43.979] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:10:44.064+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:10:44.064] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:10:44.072+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:10:44.072] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:10:44.072+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:10:44.072] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:10:44.073+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:10:44.073] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:10:44.074+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:10:44.074] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:10:44.074+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:10:44.074] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:10:44.074+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:10:44.074] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:10:44.075+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:10:44.075] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:10:44.286+0000] [.info] [src/director/director.go:tracef:794] [01:10:44.286] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:10:44.286+0000] [.info] [src/director/director.go:tracef:794] [01:10:44.286] because ['currentState.Up' = false] [2021-10-22T01:10:44.483+0000] [.info] [src/director/director.go:tracef:794] [01:10:44.483] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:10:44.484+0000] [.info] [src/director/director.go:tracef:794] [01:10:44.484] because ['currentState.Up' = false] [2021-10-22T01:10:44.484+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:10:44.484] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:10:44.531+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:44.531] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:44.720+0000] [.info] [src/director/director.go:tracef:794] [01:10:44.720] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:10:44.720+0000] [.info] [src/director/director.go:tracef:794] [01:10:44.720] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:10:44.924+0000] [.info] [src/director/director.go:tracef:794] [01:10:44.924] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:10:44.925+0000] [.info] [src/director/director.go:tracef:794] [01:10:44.924] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:10:44.925+0000] [.info] [src/director/director.go:tracef:794] [01:10:44.925] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:10:44.925+0000] [.info] [src/director/director.go:tracef:794] [01:10:44.925] because ['currentState.Up' = false] [2021-10-22T01:10:44.925+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:10:44.925] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:10:44.925+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:10:44.925] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:10:44.960+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:10:44.960] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:10:44.961+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:10:44.961] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57385 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:10:45.432+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:10:45.432] clusterConfig unchanged [2021-10-22T01:10:45.505+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:45.505] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:45.688+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:10:45.688] ...Finished start [pid=21813] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57385,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:10:45.821+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:10:45.821] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 132.810457ms. [2021-10-22T01:10:45.821+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:10:45.821] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57385 (local=false) [2021-10-22T01:10:45.821+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:10:45.821] Dropping existing documents from system.replset [2021-10-22T01:10:45.940+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:10:45.940] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:10:45.943+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:10:45.943] Inserting oplogTruncateAfterPoint document [2021-10-22T01:10:45.943+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:10:45.943] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:10:45.944+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:10:45.944] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:10:45.944+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:10:45.944] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:10:45.956+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:10:45.955] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:10:45.956+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:10:45.956] Setting the minvalid document to [{_id ObjectID("61720f95523370002becc494")} {t -1} {ts {0 1}}] [2021-10-22T01:10:45.968+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:10:45.968] Inserted [{_id ObjectID("61720f95523370002becc494")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:10:45.968+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:10:45.968] Inserting new replica set configuration document. [2021-10-22T01:10:45.968+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:10:45.968] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:10:45.981+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:10:45.981] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:10:45.987+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:10:45.987] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:10:45.987+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:10:45.987] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:10:45.987+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:10:45.987] Initiating stopEphemeralMongo [2021-10-22T01:10:45.987+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:10:45.987] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57385 (local=false) [2021-10-22T01:10:45.987+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:10:45.987] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57385 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:10:46.383+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:10:46.383] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.044661ms. [2021-10-22T01:10:46.383+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:10:46.383] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57385 (local=false) [2021-10-22T01:10:46.383+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:10:46.383] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 39.121µs. [2021-10-22T01:10:46.383+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:10:46.383] Successfully finished running stopEphemeralMongo [2021-10-22T01:10:46.383+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:10:46.383] Successfully stopped existing ephemeral mongod. [2021-10-22T01:10:46.383+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:10:46.383] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:10:46.383+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:10:46.383] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:10:46.415+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:10:46.415] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:10:46.416+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:10:46.416] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34495 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:10:46.506+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:46.506] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:46.584+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:10:46.584] clusterConfig unchanged [2021-10-22T01:10:47.509+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:47.509] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:47.596+0000] [.error] [src/action/start.go:func1:93] [103] [01:10:47.596] Error starting mongod : [01:10:47.595] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34495 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:10:46.449Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 23373 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:10:46.469+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:10:46.469+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:10:46.469+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:10:46.471+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:10:46.473+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:10:46.473+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:10:46.497+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:10:46.497+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:10:46.508+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:10:46.553+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:10:46.553+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:10:47.077+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865047:77671][23373:0x7feabf8e8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:10:47.184+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865047:184216][23373:0x7feabf8e8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:10:47.290+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865047:290798][23373:0x7feabf8e8bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:10:47.405+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865047:405818][23373:0x7feabf8e8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:10:47.480+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865047:480632][23373:0x7feabf8e8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:10:47.546+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865047:546843][23373:0x7feabf8e8bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:10:47.546+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865047:546916][23373:0x7feabf8e8bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:10:47.596+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:10:47.596] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34495 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:10:47.596] Error starting mongod : [01:10:47.595] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34495 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:10:46.449Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 23373 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:10:46.469+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:10:46.469+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:10:46.469+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:10:46.471+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:10:46.473+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:10:46.473+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:10:46.497+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:10:46.497+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:10:46.508+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:10:46.553+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:10:46.553+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:10:47.077+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865047:77671][23373:0x7feabf8e8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:10:47.184+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:10:47.596+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:10:47.596] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:10:47.596] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34495 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:10:47.596] Error starting mongod : [01:10:47.595] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34495 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:10:46.449Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 23373 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:10:46.469+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:10:46.469+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:10:46.469+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:10:46.471+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:10:46.473+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:10:46.473+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:10:46.497+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:10:46.497+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:10:46.508+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:10:46.553+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:10:46.553+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:10:47.077+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865047:77671][23373:0x7feabf8e8bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:10:47.596+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:10:47.596] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:10:47.596+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:10:47.596] Initiating stopEphemeralMongo [2021-10-22T01:10:47.596+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:10:47.596] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:10:47.597+0000] [.error] [src/director/director.go:executePlan:962] [01:10:47.597] Failed to apply action. Result = : [01:10:47.596] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:10:47.596] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34495 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:10:47.596] Error starting mongod : [01:10:47.595] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34495 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:10:46.449Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 23373 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:10:46.469+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:10:46.469+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:10:46.469+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:10:46.471+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:10:46.473+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:10:46.473+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:10:46.497+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:10:46.497+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:10:46.508+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:10:46.553+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:10:46.553+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:10:47.077+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:10:47.597+0000] [.error] [src/director/director.go:planAndExecute:567] [01:10:47.597] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:10:47.597] Failed to apply action. Result = : [01:10:47.596] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:10:47.596] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34495 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:10:47.596] Error starting mongod : [01:10:47.595] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34495 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:10:46.449Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 23373 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:10:46.469+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:10:46.469+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:10:46.469+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:10:46.471+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:10:46.473+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:10:46.473+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:10:46.497+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:10:46.497+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:10:46.508+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:10:46.553+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:10:46.553+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:10:47.597+0000] [.error] [src/director/director.go:mainLoop:399] [01:10:47.597] Failed to planAndExecute : [01:10:47.597] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:10:47.597] Failed to apply action. Result = : [01:10:47.596] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:10:47.596] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34495 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:10:47.596] Error starting mongod : [01:10:47.595] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34495 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:10:46.449Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 23373 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:10:46.469+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:10:46.469+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:10:46.469+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:10:46.471+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:10:46.473+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:10:46.473+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:10:46.497+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:10:46.497+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:10:46.508+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:10:46.553+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:10:46.553+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:10:47.834+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:10:47.834] clusterConfig unchanged [2021-10-22T01:10:48.508+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:48.508] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:49.511+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:49.511] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:50.512+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:50.512] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:51.513+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:51.513] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:52.514+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:52.514] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:53.513+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:53.513] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:54.513+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:54.513] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:55.514+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:55.514] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:56.376+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:10:56.376] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.300505008s [2021-10-22T01:10:56.512+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:56.512] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:57.512+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:57.512] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:57.946+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:10:57.946] clusterConfig unchanged [2021-10-22T01:10:58.513+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:58.513] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:10:59.514+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:10:59.514] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:00.515+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:00.515] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:01.515+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:01.515] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:02.516+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:02.516] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:03.516+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:03.516] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:04.516+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:04.516] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:05.516+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:05.516] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:06.517+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:06.517] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:07.523+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:07.523] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:08.058+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:11:08.058] clusterConfig unchanged [2021-10-22T01:11:08.518+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:08.518] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:09.534+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:09.534] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:10.519+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:10.519] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:11.529+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:11.529] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:12.521+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:12.521] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:13.521+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:13.521] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:14.523+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:14.523] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:15.523+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:15.523] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:16.523+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:16.523] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:17.525+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:17.525] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:18.150+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:11:18.150] clusterConfig unchanged [2021-10-22T01:11:18.530+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:18.530] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:18.558+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:11:18.558] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:11:18.560+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:18.560] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:11:18.560+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [01:11:18.560] Could not get version : [01:11:18.560] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:11:18.560+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [01:11:18.560] Error getting version for log lines. Skipping : [01:11:18.560] Could not get version : [01:11:18.560] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:11:19.525+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:19.525] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:20.525+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:20.525] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:21.526+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:21.526] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:22.529+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:22.529] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:23.527+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:23.527] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:24.527+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:24.527] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:25.528+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:25.528] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:26.529+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:26.529] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:27.530+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:27.530] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:28.262+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:11:28.262] clusterConfig unchanged [2021-10-22T01:11:28.530+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:28.530] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:29.531+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:29.531] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:30.531+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:30.531] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:31.532+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:31.532] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:32.532+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:32.532] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:33.532+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:33.532] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:34.534+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:34.534] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:35.534+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:35.534] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:36.534+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:36.534] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:37.534+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:37.534] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:38.394+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:11:38.394] clusterConfig unchanged [2021-10-22T01:11:38.535+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:38.535] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:39.541+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:39.541] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:40.536+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:40.536] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:41.311+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:11:41.311] memory:rss=51871744 alloc=8028552 totalalloc=4362024520 mallocs=55871673 frees=55815730 heapinuse=14204928 heapobjects=55943 gcpausetotalns=1456980121 gcpauselastns=63212 gcnum=692 utimens=79600156000 stimens=26761940000 minflt=216404 majflt=405 [2021-10-22T01:11:41.537+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:41.537] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:42.538+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:42.537] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:43.538+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:43.538] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:44.218+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [01:11:44.218] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34495 (local=false) because it is not up [2021-10-22T01:11:44.539+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:44.539] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:45.544+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:45.544] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:46.540+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:46.540] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:47.540+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:47.540] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:47.754+0000] [.info] [src/director/director.go:computePlan:280] [01:11:47.754] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:11:47.754+0000] [.info] [src/director/director.go:tracef:794] [01:11:47.754] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:11:47.754+0000] [.info] [src/director/director.go:tracef:794] [01:11:47.754] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:11:47.754+0000] [.info] [src/director/director.go:tracef:794] [01:11:47.754] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:11:47.754+0000] [.info] [src/director/director.go:tracef:794] [01:11:47.754] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:11:47.754+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:11:47.754] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:11:47.914+0000] [.info] [src/director/director.go:tracef:794] [01:11:47.914] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:11:47.914+0000] [.info] [src/director/director.go:tracef:794] [01:11:47.914] because [2021-10-22T01:11:47.915+0000] [.info] [src/director/director.go:tracef:794] [01:11:47.915] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:11:47.915+0000] [.info] [src/director/director.go:tracef:794] [01:11:47.915] because ['currentState.CanBounceStop' = true] [2021-10-22T01:11:48.094+0000] [.info] [src/director/director.go:tracef:794] [01:11:48.094] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:11:48.094+0000] [.info] [src/director/director.go:tracef:794] [01:11:48.094] because [2021-10-22T01:11:48.094+0000] [.info] [src/director/director.go:tracef:794] [01:11:48.094] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:11:48.094+0000] [.info] [src/director/director.go:tracef:794] [01:11:48.094] because ['currentState.Up' = false] [2021-10-22T01:11:48.095+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:11:48.095] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:11:48.095+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:11:48.095] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:11:48.181+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:11:48.181] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:11:48.181+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:11:48.181] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:11:48.189+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:11:48.189] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:11:48.190+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:11:48.190] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:11:48.191+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:11:48.191] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:11:48.191+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:11:48.191] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:11:48.191+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:11:48.191] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:11:48.192+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:11:48.192] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:11:48.404+0000] [.info] [src/director/director.go:tracef:794] [01:11:48.404] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:11:48.404+0000] [.info] [src/director/director.go:tracef:794] [01:11:48.404] because ['currentState.Up' = false] [2021-10-22T01:11:48.544+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:48.544] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:48.575+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:11:48.575] clusterConfig unchanged [2021-10-22T01:11:48.604+0000] [.info] [src/director/director.go:tracef:794] [01:11:48.604] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:11:48.604+0000] [.info] [src/director/director.go:tracef:794] [01:11:48.604] because ['currentState.Up' = false] [2021-10-22T01:11:48.604+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:11:48.604] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:11:48.796+0000] [.info] [src/director/director.go:tracef:794] [01:11:48.796] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:11:48.796+0000] [.info] [src/director/director.go:tracef:794] [01:11:48.796] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:11:49.012+0000] [.info] [src/director/director.go:tracef:794] [01:11:49.012] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:11:49.012+0000] [.info] [src/director/director.go:tracef:794] [01:11:49.012] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:11:49.013+0000] [.info] [src/director/director.go:tracef:794] [01:11:49.013] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:11:49.013+0000] [.info] [src/director/director.go:tracef:794] [01:11:49.013] because ['currentState.Up' = false] [2021-10-22T01:11:49.013+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:11:49.013] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:11:49.013+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:11:49.013] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:11:49.044+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:11:49.044] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:11:49.044+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:11:49.044] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49782 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:11:49.546+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:49.546] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:49.666+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:11:49.666] clusterConfig unchanged [2021-10-22T01:11:49.760+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:11:49.760] ...Finished start [pid=42443] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49782,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:11:49.885+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:11:49.885] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 125.331607ms. [2021-10-22T01:11:49.885+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:11:49.885] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49782 (local=false) [2021-10-22T01:11:49.885+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:11:49.885] Dropping existing documents from system.replset [2021-10-22T01:11:50.041+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:11:50.041] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:11:50.044+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:11:50.044] Inserting oplogTruncateAfterPoint document [2021-10-22T01:11:50.044+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:11:50.044] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:11:50.045+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:11:50.045] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:11:50.045+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:11:50.045] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:11:50.059+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:11:50.059] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:11:50.059+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:11:50.059] Setting the minvalid document to [{_id ObjectID("61720fd6523370002becc4a2")} {t -1} {ts {0 1}}] [2021-10-22T01:11:50.072+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:11:50.072] Inserted [{_id ObjectID("61720fd6523370002becc4a2")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:11:50.072+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:11:50.072] Inserting new replica set configuration document. [2021-10-22T01:11:50.072+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:11:50.072] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:11:50.086+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:11:50.086] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:11:50.089+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:11:50.089] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:11:50.089+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:11:50.089] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:11:50.089+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:11:50.089] Initiating stopEphemeralMongo [2021-10-22T01:11:50.089+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:11:50.089] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49782 (local=false) [2021-10-22T01:11:50.089+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:11:50.089] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49782 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:11:50.467+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:11:50.467] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.104144ms. [2021-10-22T01:11:50.467+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:11:50.467] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49782 (local=false) [2021-10-22T01:11:50.467+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:11:50.467] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 40.614µs. [2021-10-22T01:11:50.467+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:11:50.467] Successfully finished running stopEphemeralMongo [2021-10-22T01:11:50.467+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:11:50.467] Successfully stopped existing ephemeral mongod. [2021-10-22T01:11:50.467+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:11:50.467] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:11:50.467+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:11:50.467] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:11:50.501+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:11:50.501] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:11:50.502+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:11:50.502] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58620 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:11:50.566+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:50.566] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:50.839+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:11:50.839] clusterConfig unchanged [2021-10-22T01:11:51.561+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:51.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:51.757+0000] [.error] [src/action/start.go:func1:93] [103] [01:11:51.757] Error starting mongod : [01:11:51.757] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58620,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58620 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:11:50.534Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 43993 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:11:50.561+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:11:50.561+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:11:50.561+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58620,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:11:50.563+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:11:50.567+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:11:50.578+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:11:50.604+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:11:50.604+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:11:50.606+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:11:50.651+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:11:50.651+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:11:51.219+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865111:219772][43993:0x7f721cf0ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:11:51.321+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865111:321773][43993:0x7f721cf0ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:11:51.430+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865111:430895][43993:0x7f721cf0ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:11:51.565+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865111:565852][43993:0x7f721cf0ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:11:51.642+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865111:642425][43993:0x7f721cf0ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:11:51.702+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865111:702670][43993:0x7f721cf0ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:11:51.702+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865111:702741][43993:0x7f721cf0ebc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:11:51.757+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:11:51.757] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:58620 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:11:51.757] Error starting mongod : [01:11:51.757] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58620,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58620 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:11:50.534Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 43993 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:11:50.561+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:11:50.561+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:11:50.561+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58620,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:11:50.563+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:11:50.567+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:11:50.578+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:11:50.604+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:11:50.604+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:11:50.606+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:11:50.651+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:11:50.651+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:11:51.219+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865111:219772][43993:0x7f721cf0ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:11:51.321+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:11:51.757+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:11:51.757] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:11:51.757] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:58620 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:11:51.757] Error starting mongod : [01:11:51.757] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58620,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58620 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:11:50.534Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 43993 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:11:50.561+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:11:50.561+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:11:50.561+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58620,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:11:50.563+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:11:50.567+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:11:50.578+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:11:50.604+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:11:50.604+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:11:50.606+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:11:50.651+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:11:50.651+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:11:51.219+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865111:219772][43993:0x7f721cf0ebc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:11:51.758+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:11:51.758] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:11:51.758+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:11:51.758] Initiating stopEphemeralMongo [2021-10-22T01:11:51.758+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:11:51.758] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:11:51.758+0000] [.error] [src/director/director.go:executePlan:962] [01:11:51.758] Failed to apply action. Result = : [01:11:51.757] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:11:51.757] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:58620 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:11:51.757] Error starting mongod : [01:11:51.757] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58620,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58620 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:11:50.534Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 43993 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:11:50.561+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:11:50.561+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:11:50.561+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58620,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:11:50.563+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:11:50.567+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:11:50.578+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:11:50.604+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:11:50.604+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:11:50.606+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:11:50.651+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:11:50.651+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:11:51.219+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:11:51.759+0000] [.error] [src/director/director.go:planAndExecute:567] [01:11:51.758] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:11:51.758] Failed to apply action. Result = : [01:11:51.757] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:11:51.757] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:58620 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:11:51.757] Error starting mongod : [01:11:51.757] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58620,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58620 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:11:50.534Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 43993 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:11:50.561+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:11:50.561+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:11:50.561+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58620,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:11:50.563+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:11:50.567+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:11:50.578+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:11:50.604+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:11:50.604+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:11:50.606+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:11:50.651+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:11:50.651+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:11:51.759+0000] [.error] [src/director/director.go:mainLoop:399] [01:11:51.759] Failed to planAndExecute : [01:11:51.758] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:11:51.758] Failed to apply action. Result = : [01:11:51.757] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:11:51.757] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:58620 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:11:51.757] Error starting mongod : [01:11:51.757] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58620,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58620 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:11:50.534Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 43993 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:11:50.561+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:11:50.561+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:11:50.561+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58620,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:11:50.563+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:11:50.567+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:11:50.578+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:11:50.604+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:11:50.604+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:11:50.606+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:11:50.651+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:11:50.651+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:11:52.079+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:11:52.079] clusterConfig unchanged [2021-10-22T01:11:52.561+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:52.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:53.561+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:53.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:54.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:54.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:55.563+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:55.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:56.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:56.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:57.564+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:57.564] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:58.565+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:58.565] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:11:59.564+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:11:59.564] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:00.539+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:12:00.539] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.34691409s [2021-10-22T01:12:00.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:00.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:01.563+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:01.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:02.222+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:12:02.222] clusterConfig unchanged [2021-10-22T01:12:02.565+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:02.565] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:03.564+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:03.564] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:04.570+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:04.570] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:05.565+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:05.565] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:06.566+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:06.566] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:07.567+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:07.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:08.567+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:08.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:09.567+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:09.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:10.567+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:10.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:11.568+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:11.568] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:12.362+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:12:12.362] clusterConfig unchanged [2021-10-22T01:12:12.569+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:12.569] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:13.569+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:13.569] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:14.570+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:14.570] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:15.572+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:15.572] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:16.571+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:16.571] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:17.572+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:17.572] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:18.572+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:18.572] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:19.573+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:19.572] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:20.573+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:20.573] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:21.574+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:21.574] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:22.474+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:12:22.474] clusterConfig unchanged [2021-10-22T01:12:22.574+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:22.574] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:23.575+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:23.575] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:24.575+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:24.575] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:25.577+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:25.577] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:26.577+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:26.577] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:27.560+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:12:27.560] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:12:27.562+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:12:27.562] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:12:27.579+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:27.579] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:28.579+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:28.579] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:29.579+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:29.579] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:30.580+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:30.580] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:31.581+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:31.581] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:32.581+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:32.581] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:32.586+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:12:32.586] clusterConfig unchanged [2021-10-22T01:12:33.582+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:33.582] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:34.582+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:34.582] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:35.585+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [01:12:35.585] lsblk cache invalidated, running lsblk command [2021-10-22T01:12:35.586+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:35.586] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:36.583+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:36.583] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:37.584+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:37.584] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:38.591+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:38.591] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:39.585+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:39.585] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:40.605+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:40.605] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:41.312+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:12:41.312] memory:rss=51748864 alloc=8540360 totalalloc=4449970912 mallocs=56981111 frees=56919648 heapinuse=14237696 heapobjects=61463 gcpausetotalns=1458529967 gcpauselastns=77869 gcnum=706 utimens=81414274000 stimens=27439513000 minflt=221087 majflt=405 [2021-10-22T01:12:41.586+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:41.586] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:42.588+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:42.588] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:42.698+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:12:42.698] clusterConfig unchanged [2021-10-22T01:12:43.589+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:43.589] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:44.589+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:44.589] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:45.590+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:45.590] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:46.605+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:46.605] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:47.592+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:47.592] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:48.592+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:48.592] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:49.592+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:49.592] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:50.593+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:50.593] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:51.594+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:51.594] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:51.924+0000] [.info] [src/director/director.go:computePlan:280] [01:12:51.924] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:12:51.924+0000] [.info] [src/director/director.go:tracef:794] [01:12:51.924] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:12:51.924+0000] [.info] [src/director/director.go:tracef:794] [01:12:51.924] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:12:51.924+0000] [.info] [src/director/director.go:tracef:794] [01:12:51.924] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:12:51.924+0000] [.info] [src/director/director.go:tracef:794] [01:12:51.924] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:12:51.924+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:12:51.924] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:12:52.079+0000] [.info] [src/director/director.go:tracef:794] [01:12:52.079] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:12:52.079+0000] [.info] [src/director/director.go:tracef:794] [01:12:52.079] because [2021-10-22T01:12:52.079+0000] [.info] [src/director/director.go:tracef:794] [01:12:52.079] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:12:52.079+0000] [.info] [src/director/director.go:tracef:794] [01:12:52.079] because ['currentState.CanBounceStop' = true] [2021-10-22T01:12:52.234+0000] [.info] [src/director/director.go:tracef:794] [01:12:52.234] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:12:52.234+0000] [.info] [src/director/director.go:tracef:794] [01:12:52.234] because [2021-10-22T01:12:52.234+0000] [.info] [src/director/director.go:tracef:794] [01:12:52.234] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:12:52.234+0000] [.info] [src/director/director.go:tracef:794] [01:12:52.234] because ['currentState.Up' = false] [2021-10-22T01:12:52.234+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:12:52.234] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:12:52.234+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:12:52.234] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:12:52.318+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:12:52.318] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:12:52.326+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:12:52.326] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:12:52.326+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:12:52.326] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:12:52.327+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:12:52.327] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:12:52.328+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:12:52.328] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:12:52.328+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:12:52.328] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:12:52.328+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:12:52.328] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:12:52.329+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:12:52.329] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:12:52.521+0000] [.info] [src/director/director.go:tracef:794] [01:12:52.521] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:12:52.521+0000] [.info] [src/director/director.go:tracef:794] [01:12:52.521] because ['currentState.Up' = false] [2021-10-22T01:12:52.610+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:52.610] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:52.741+0000] [.info] [src/director/director.go:tracef:794] [01:12:52.741] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:12:52.741+0000] [.info] [src/director/director.go:tracef:794] [01:12:52.741] because ['currentState.Up' = false] [2021-10-22T01:12:52.742+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:12:52.742] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:12:52.930+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:12:52.930] clusterConfig unchanged [2021-10-22T01:12:52.967+0000] [.info] [src/director/director.go:tracef:794] [01:12:52.967] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:12:52.967+0000] [.info] [src/director/director.go:tracef:794] [01:12:52.967] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:12:53.175+0000] [.info] [src/director/director.go:tracef:794] [01:12:53.175] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:12:53.175+0000] [.info] [src/director/director.go:tracef:794] [01:12:53.175] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:12:53.176+0000] [.info] [src/director/director.go:tracef:794] [01:12:53.176] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:12:53.176+0000] [.info] [src/director/director.go:tracef:794] [01:12:53.176] because ['currentState.Up' = false] [2021-10-22T01:12:53.176+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:12:53.176] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:12:53.176+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:12:53.176] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:12:53.212+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:12:53.212] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:12:53.213+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:12:53.213] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51322 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:12:53.600+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:53.600] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:53.929+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:12:53.929] ...Finished start [pid=63310] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51322,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:12:54.045+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:12:54.045] clusterConfig unchanged [2021-10-22T01:12:54.105+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:12:54.105] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 176.036232ms. [2021-10-22T01:12:54.105+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:12:54.105] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51322 (local=false) [2021-10-22T01:12:54.105+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:12:54.105] Dropping existing documents from system.replset [2021-10-22T01:12:54.305+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:12:54.305] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:12:54.308+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:12:54.308] Inserting oplogTruncateAfterPoint document [2021-10-22T01:12:54.308+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:12:54.308] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:12:54.309+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:12:54.309] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:12:54.309+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:12:54.309] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:12:54.323+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:12:54.323] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:12:54.323+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:12:54.323] Setting the minvalid document to [{_id ObjectID("61721016523370002becc4b1")} {t -1} {ts {0 1}}] [2021-10-22T01:12:54.335+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:12:54.335] Inserted [{_id ObjectID("61721016523370002becc4b1")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:12:54.336+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:12:54.335] Inserting new replica set configuration document. [2021-10-22T01:12:54.336+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:12:54.336] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:12:54.349+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:12:54.349] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:12:54.363+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:12:54.363] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:12:54.363+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:12:54.363] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:12:54.363+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:12:54.363] Initiating stopEphemeralMongo [2021-10-22T01:12:54.363+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:12:54.363] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51322 (local=false) [2021-10-22T01:12:54.363+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:12:54.363] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51322 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:12:54.600+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:54.599] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:54.818+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:12:54.818] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.083368ms. [2021-10-22T01:12:54.818+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:12:54.818] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51322 (local=false) [2021-10-22T01:12:54.818+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:12:54.818] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.912µs. [2021-10-22T01:12:54.818+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:12:54.818] Successfully finished running stopEphemeralMongo [2021-10-22T01:12:54.818+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:12:54.818] Successfully stopped existing ephemeral mongod. [2021-10-22T01:12:54.818+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:12:54.818] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:12:54.818+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:12:54.818] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:12:54.854+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:12:54.854] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:12:54.855+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:12:54.854] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57897 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:12:55.183+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:12:55.183] clusterConfig unchanged [2021-10-22T01:12:55.601+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:55.601] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:56.025+0000] [.error] [src/action/start.go:func1:93] [103] [01:12:56.025] Error starting mongod : [01:12:56.024] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57897,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57897 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 64972 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:12:54.909+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:12:54.909+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:12:54.909+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57897,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:12:54.910+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:12:54.912+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:12:54.913+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:12:54.933+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:12:54.933+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:12:54.933+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:12:54.979+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:12:54.979+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:12:55.516+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865175:516001][64972:0x7fa306648bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:12:55.622+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865175:622752][64972:0x7fa306648bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:12:55.725+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865175:725375][64972:0x7fa306648bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:12:55.839+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865175:839094][64972:0x7fa306648bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:12:55.912+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865175:912888][64972:0x7fa306648bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:12:55.971+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865175:970990][64972:0x7fa306648bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:12:55.971+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865175:971065][64972:0x7fa306648bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:12:55.972+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:12:56.025+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:12:56.025] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:57897 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:12:56.025] Error starting mongod : [01:12:56.024] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57897,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57897 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 64972 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:12:54.909+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:12:54.909+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:12:54.909+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57897,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:12:54.910+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:12:54.912+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:12:54.913+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:12:54.933+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:12:54.933+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:12:54.933+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:12:54.979+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:12:54.979+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:12:55.516+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865175:516001][64972:0x7fa306648bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:12:55.622+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865175:622752][64972:0x7fa306648bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:12:56.025+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:12:56.025] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:12:56.025] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:57897 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:12:56.025] Error starting mongod : [01:12:56.024] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57897,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57897 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 64972 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:12:54.909+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:12:54.909+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:12:54.909+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57897,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:12:54.910+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:12:54.912+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:12:54.913+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:12:54.933+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:12:54.933+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:12:54.933+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:12:54.979+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:12:54.979+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:12:55.516+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865175:516001][64972:0x7fa306648bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:12:55.622+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:12:56.025+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:12:56.025] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:12:56.025+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:12:56.025] Initiating stopEphemeralMongo [2021-10-22T01:12:56.025+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:12:56.025] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:12:56.025+0000] [.error] [src/director/director.go:executePlan:962] [01:12:56.025] Failed to apply action. Result = : [01:12:56.025] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:12:56.025] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:57897 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:12:56.025] Error starting mongod : [01:12:56.024] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57897,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57897 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 64972 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:12:54.909+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:12:54.909+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:12:54.909+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57897,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:12:54.910+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:12:54.912+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:12:54.913+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:12:54.933+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:12:54.933+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:12:54.933+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:12:54.979+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:12:54.979+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:12:55.516+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865175:516001][64972:0x7fa306648bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:12:56.026+0000] [.error] [src/director/director.go:planAndExecute:567] [01:12:56.025] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:12:56.025] Failed to apply action. Result = : [01:12:56.025] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:12:56.025] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:57897 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:12:56.025] Error starting mongod : [01:12:56.024] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57897,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57897 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 64972 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:12:54.909+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:12:54.909+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:12:54.909+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57897,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:12:54.910+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:12:54.912+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:12:54.913+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:12:54.933+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:12:54.933+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:12:54.933+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:12:54.979+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:12:54.979+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:12:55.516+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:12:56.026+0000] [.error] [src/director/director.go:mainLoop:399] [01:12:56.026] Failed to planAndExecute : [01:12:56.025] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:12:56.025] Failed to apply action. Result = : [01:12:56.025] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:12:56.025] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:57897 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:12:56.025] Error starting mongod : [01:12:56.024] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57897,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57897 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 64972 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:12:54.909+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:12:54.909+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:12:54.909+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57897,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:12:54.910+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:12:54.912+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:12:54.913+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:12:54.933+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:12:54.933+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:12:54.933+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:12:54.979+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:12:54.979+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:12:55.516+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:12:56.424+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:12:56.424] clusterConfig unchanged [2021-10-22T01:12:56.603+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:56.603] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:57.600+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:57.600] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:58.606+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:58.606] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:12:59.601+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:12:59.601] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:00.603+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:00.603] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:01.605+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:01.605] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:02.604+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:02.604] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:03.604+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:03.604] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:04.603+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:04.603] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:04.763+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:13:04.763] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.433614142s [2021-10-22T01:13:05.611+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:05.611] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:06.555+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:13:06.555] clusterConfig unchanged [2021-10-22T01:13:06.605+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:06.605] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:07.604+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:07.604] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:08.604+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:08.604] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:09.610+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:09.610] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:10.605+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:10.605] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:11.606+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:11.606] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:12.606+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:12.606] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:13.607+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:13.607] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:14.607+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:14.607] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:15.608+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:15.608] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:16.609+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:16.609] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:16.657+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:13:16.657] clusterConfig unchanged [2021-10-22T01:13:17.609+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:17.609] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:18.609+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:18.609] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:19.610+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:19.610] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:20.616+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:20.616] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:21.610+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:21.610] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:22.611+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:22.611] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:23.628+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:23.628] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:24.612+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:24.612] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:25.613+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:25.613] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:26.615+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:26.615] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:26.789+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:13:26.789] clusterConfig unchanged [2021-10-22T01:13:27.615+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:27.615] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:28.615+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:28.615] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:29.616+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:29.616] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:30.616+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:30.616] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:31.617+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:31.617] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:32.624+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:32.624] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:33.619+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:33.619] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:34.619+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:34.619] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:35.620+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:35.620] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:36.620+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:36.620] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:36.911+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:13:36.911] clusterConfig unchanged [2021-10-22T01:13:37.625+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:37.625] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:38.621+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:38.621] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:39.621+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:39.621] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:40.622+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:40.622] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:41.313+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:13:41.313] memory:rss=51752960 alloc=11604528 totalalloc=4535218736 mallocs=58068782 frees=57974231 heapinuse=15351808 heapobjects=94551 gcpausetotalns=1460597101 gcpauselastns=74532 gcnum=718 utimens=83159829000 stimens=28092579000 minflt=225793 majflt=405 [2021-10-22T01:13:41.623+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:41.623] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:42.623+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:42.623] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:43.623+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:43.623] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:44.624+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:44.624] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:45.625+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:45.625] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:46.626+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:46.626] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:47.052+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:13:47.052] clusterConfig unchanged [2021-10-22T01:13:47.625+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:47.625] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:48.627+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:48.627] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:49.627+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:49.627] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:50.627+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:50.627] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:51.628+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:51.628] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:52.629+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:52.629] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:53.636+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:53.636] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:54.630+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:54.630] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:55.631+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:55.631] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:56.183+0000] [.info] [src/director/director.go:computePlan:280] [01:13:56.183] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:13:56.183+0000] [.info] [src/director/director.go:tracef:794] [01:13:56.183] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:13:56.183+0000] [.info] [src/director/director.go:tracef:794] [01:13:56.183] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:13:56.183+0000] [.info] [src/director/director.go:tracef:794] [01:13:56.183] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:13:56.183+0000] [.info] [src/director/director.go:tracef:794] [01:13:56.183] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:13:56.183+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:13:56.183] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:13:56.338+0000] [.info] [src/director/director.go:tracef:794] [01:13:56.338] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:13:56.338+0000] [.info] [src/director/director.go:tracef:794] [01:13:56.338] because [2021-10-22T01:13:56.338+0000] [.info] [src/director/director.go:tracef:794] [01:13:56.338] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:13:56.338+0000] [.info] [src/director/director.go:tracef:794] [01:13:56.338] because ['currentState.CanBounceStop' = true] [2021-10-22T01:13:56.493+0000] [.info] [src/director/director.go:tracef:794] [01:13:56.493] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:13:56.493+0000] [.info] [src/director/director.go:tracef:794] [01:13:56.493] because [2021-10-22T01:13:56.493+0000] [.info] [src/director/director.go:tracef:794] [01:13:56.493] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:13:56.493+0000] [.info] [src/director/director.go:tracef:794] [01:13:56.493] because ['currentState.Up' = false] [2021-10-22T01:13:56.494+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:13:56.494] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:13:56.494+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:13:56.494] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:13:56.574+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:13:56.574] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:13:56.585+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:13:56.585] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:13:56.586+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:13:56.586] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:13:56.586+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:13:56.586] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:13:56.588+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:13:56.588] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:13:56.588+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:13:56.588] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:13:56.588+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:13:56.588] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:13:56.589+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:13:56.589] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:13:56.641+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:56.641] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:56.815+0000] [.info] [src/director/director.go:tracef:794] [01:13:56.815] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:13:56.815+0000] [.info] [src/director/director.go:tracef:794] [01:13:56.815] because ['currentState.Up' = false] [2021-10-22T01:13:57.023+0000] [.info] [src/director/director.go:tracef:794] [01:13:57.023] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:13:57.023+0000] [.info] [src/director/director.go:tracef:794] [01:13:57.023] because ['currentState.Up' = false] [2021-10-22T01:13:57.023+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:13:57.023] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:13:57.229+0000] [.info] [src/director/director.go:tracef:794] [01:13:57.229] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:13:57.229+0000] [.info] [src/director/director.go:tracef:794] [01:13:57.229] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:13:57.273+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:13:57.273] clusterConfig unchanged [2021-10-22T01:13:57.437+0000] [.info] [src/director/director.go:tracef:794] [01:13:57.437] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:13:57.437+0000] [.info] [src/director/director.go:tracef:794] [01:13:57.437] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:13:57.437+0000] [.info] [src/director/director.go:tracef:794] [01:13:57.437] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:13:57.437+0000] [.info] [src/director/director.go:tracef:794] [01:13:57.437] because ['currentState.Up' = false] [2021-10-22T01:13:57.437+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:13:57.437] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:13:57.437+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:13:57.437] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:13:57.473+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:13:57.473] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:13:57.473+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:13:57.473] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52493 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:13:57.641+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:57.641] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:58.199+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:13:58.199] ...Finished start [pid=10053] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52493,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:13:58.307+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:13:58.307] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 108.166902ms. [2021-10-22T01:13:58.307+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:13:58.307] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52493 (local=false) [2021-10-22T01:13:58.307+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:13:58.307] Dropping existing documents from system.replset [2021-10-22T01:13:58.410+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:13:58.410] clusterConfig unchanged [2021-10-22T01:13:58.435+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:13:58.435] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:13:58.439+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:13:58.439] Inserting oplogTruncateAfterPoint document [2021-10-22T01:13:58.439+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:13:58.439] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:13:58.440+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:13:58.440] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:13:58.440+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:13:58.440] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:13:58.453+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:13:58.453] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:13:58.453+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:13:58.453] Setting the minvalid document to [{_id ObjectID("61721056523370002becc4bf")} {t -1} {ts {0 1}}] [2021-10-22T01:13:58.464+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:13:58.464] Inserted [{_id ObjectID("61721056523370002becc4bf")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:13:58.464+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:13:58.464] Inserting new replica set configuration document. [2021-10-22T01:13:58.464+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:13:58.464] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:13:58.477+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:13:58.477] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:13:58.493+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:13:58.493] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:13:58.493+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:13:58.493] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:13:58.493+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:13:58.493] Initiating stopEphemeralMongo [2021-10-22T01:13:58.493+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:13:58.493] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52493 (local=false) [2021-10-22T01:13:58.493+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:13:58.493] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52493 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:13:58.641+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:58.641] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:13:58.915+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:13:58.915] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.72731ms. [2021-10-22T01:13:58.915+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:13:58.915] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52493 (local=false) [2021-10-22T01:13:58.915+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:13:58.915] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.594µs. [2021-10-22T01:13:58.915+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:13:58.915] Successfully finished running stopEphemeralMongo [2021-10-22T01:13:58.915+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:13:58.915] Successfully stopped existing ephemeral mongod. [2021-10-22T01:13:58.916+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:13:58.915] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:13:58.916+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:13:58.916] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:13:58.949+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:13:58.949] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:13:58.949+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:13:58.949] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34323 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:13:59.558+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:13:59.558] clusterConfig unchanged [2021-10-22T01:13:59.658+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:13:59.658] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:00.234+0000] [.error] [src/action/start.go:func1:93] [103] [01:14:00.234] Error starting mongod : [01:14:00.233] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34323,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34323 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:13:58.981Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 11642 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:13:59.001+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:13:59.001+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:13:59.001+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34323,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:13:59.002+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:13:59.004+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:13:59.004+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:13:59.042+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:13:59.042+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:13:59.053+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:13:59.098+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:13:59.098+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:13:59.597+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865239:597357][11642:0x7fea38625bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:13:59.754+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865239:754752][11642:0x7fea38625bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:13:59.865+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865239:865151][11642:0x7fea38625bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:14:00.029+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865240:29953][11642:0x7fea38625bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:14:00.105+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865240:105591][11642:0x7fea38625bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:14:00.178+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865240:178201][11642:0x7fea38625bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:14:00.178+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865240:178278][11642:0x7fea38625bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:14:00.234+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:14:00.234] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34323 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:14:00.234] Error starting mongod : [01:14:00.233] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34323,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34323 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:13:58.981Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 11642 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:13:59.001+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:13:59.001+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:13:59.001+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34323,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:13:59.002+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:13:59.004+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:13:59.004+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:13:59.042+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:13:59.042+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:13:59.053+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:13:59.098+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:13:59.098+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:13:59.597+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865239:597357][11642:0x7fea38625bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:13:59.754+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:14:00.234+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:14:00.234] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:14:00.234] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34323 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:14:00.234] Error starting mongod : [01:14:00.233] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34323,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34323 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:13:58.981Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 11642 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:13:59.001+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:13:59.001+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:13:59.001+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34323,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:13:59.002+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:13:59.004+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:13:59.004+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:13:59.042+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:13:59.042+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:13:59.053+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:13:59.098+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:13:59.098+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:13:59.597+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865239:597357][11642:0x7fea38625bc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:14:00.234+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:14:00.234] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:14:00.234+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:14:00.234] Initiating stopEphemeralMongo [2021-10-22T01:14:00.234+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:14:00.234] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:14:00.235+0000] [.error] [src/director/director.go:executePlan:962] [01:14:00.235] Failed to apply action. Result = : [01:14:00.234] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:14:00.234] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34323 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:14:00.234] Error starting mongod : [01:14:00.233] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34323,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34323 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:13:58.981Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 11642 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:13:59.001+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:13:59.001+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:13:59.001+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34323,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:13:59.002+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:13:59.004+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:13:59.004+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:13:59.042+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:13:59.042+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:13:59.053+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:13:59.098+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:13:59.098+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:13:59.597+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:14:00.235+0000] [.error] [src/director/director.go:planAndExecute:567] [01:14:00.235] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:14:00.235] Failed to apply action. Result = : [01:14:00.234] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:14:00.234] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34323 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:14:00.234] Error starting mongod : [01:14:00.233] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34323,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34323 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:13:58.981Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 11642 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:13:59.001+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:13:59.001+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:13:59.001+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34323,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:13:59.002+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:13:59.004+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:13:59.004+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:13:59.042+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:13:59.042+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:13:59.053+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:13:59.098+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:13:59.098+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:14:00.235+0000] [.error] [src/director/director.go:mainLoop:399] [01:14:00.235] Failed to planAndExecute : [01:14:00.235] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:14:00.235] Failed to apply action. Result = : [01:14:00.234] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:14:00.234] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34323 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:14:00.234] Error starting mongod : [01:14:00.233] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34323,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34323 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:13:58.981Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 11642 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:13:59.001+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:13:59.001+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:13:59.001+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34323,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:13:59.002+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:13:59.004+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:13:59.004+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:13:59.042+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:13:59.042+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:13:59.053+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:13:59.098+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:13:59.098+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:14:00.645+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:00.645] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:00.827+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:14:00.827] clusterConfig unchanged [2021-10-22T01:14:01.649+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:01.649] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:02.648+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:02.648] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:03.648+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:03.648] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:04.646+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:04.646] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:05.663+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:05.663] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:06.649+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:06.649] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:07.649+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:07.649] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:08.648+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:08.648] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:08.923+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:14:08.923] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.333500517s [2021-10-22T01:14:09.650+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:09.650] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:10.649+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:10.649] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:10.959+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:14:10.959] clusterConfig unchanged [2021-10-22T01:14:11.650+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:11.650] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:12.651+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:12.651] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:13.652+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:13.652] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:14.652+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:14.652] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:15.653+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:15.653] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:16.675+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:16.675] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:17.655+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:17.655] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:18.655+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:18.655] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:19.656+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:19.656] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:20.656+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:20.656] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:21.094+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:14:21.094] clusterConfig unchanged [2021-10-22T01:14:21.656+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:21.656] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:22.657+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:22.657] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:23.657+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:23.657] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:24.658+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:24.658] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:25.658+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:25.658] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:26.660+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:26.660] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:27.663+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:27.663] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:28.661+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:28.661] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:29.662+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:29.662] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:30.662+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:30.662] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:31.242+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:14:31.242] clusterConfig unchanged [2021-10-22T01:14:31.663+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:31.663] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:32.663+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:32.663] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:33.666+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:33.666] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:34.664+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:34.664] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:35.665+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:35.665] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:36.665+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:36.665] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:37.666+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:37.666] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:38.666+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:38.666] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:39.672+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:39.672] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:40.667+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:40.667] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:41.314+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:14:41.314] memory:rss=52248576 alloc=7935008 totalalloc=4621791632 mallocs=59162088 frees=59106455 heapinuse=14303232 heapobjects=55633 gcpausetotalns=1473705345 gcpauselastns=122524 gcnum=732 utimens=84848254000 stimens=28699517000 minflt=230282 majflt=405 [2021-10-22T01:14:41.373+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:14:41.373] clusterConfig unchanged [2021-10-22T01:14:41.668+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:41.668] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:42.668+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:42.668] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:43.669+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:43.669] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:44.226+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [01:14:44.226] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34323 (local=false) because it is not up [2021-10-22T01:14:44.669+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:44.669] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:45.670+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:45.670] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:46.676+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:46.676] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:47.671+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:47.671] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:48.672+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:48.672] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:49.673+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:49.673] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:50.674+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:50.674] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:51.496+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:14:51.496] clusterConfig unchanged [2021-10-22T01:14:51.674+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:51.674] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:52.675+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:52.675] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:53.678+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:53.678] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:54.675+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:54.675] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:55.676+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:55.676] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:56.678+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:56.678] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:57.681+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:57.681] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:58.679+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:58.679] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:14:59.679+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:14:59.679] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:00.460+0000] [.info] [src/director/director.go:computePlan:280] [01:15:00.460] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:15:00.460+0000] [.info] [src/director/director.go:tracef:794] [01:15:00.460] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:15:00.460+0000] [.info] [src/director/director.go:tracef:794] [01:15:00.460] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:15:00.460+0000] [.info] [src/director/director.go:tracef:794] [01:15:00.460] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:15:00.460+0000] [.info] [src/director/director.go:tracef:794] [01:15:00.460] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:15:00.461+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:15:00.461] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:15:00.654+0000] [.info] [src/director/director.go:tracef:794] [01:15:00.654] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:15:00.654+0000] [.info] [src/director/director.go:tracef:794] [01:15:00.654] because [2021-10-22T01:15:00.654+0000] [.info] [src/director/director.go:tracef:794] [01:15:00.654] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:15:00.654+0000] [.info] [src/director/director.go:tracef:794] [01:15:00.654] because ['currentState.CanBounceStop' = true] [2021-10-22T01:15:00.685+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:00.685] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:00.882+0000] [.info] [src/director/director.go:tracef:794] [01:15:00.882] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:15:00.882+0000] [.info] [src/director/director.go:tracef:794] [01:15:00.882] because [2021-10-22T01:15:00.882+0000] [.info] [src/director/director.go:tracef:794] [01:15:00.882] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:15:00.882+0000] [.info] [src/director/director.go:tracef:794] [01:15:00.882] because ['currentState.Up' = false] [2021-10-22T01:15:00.882+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:15:00.882] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:15:00.882+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:15:00.882] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:15:00.968+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:15:00.968] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:15:00.976+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:15:00.976] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:15:00.976+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:15:00.976] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:15:00.978+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:15:00.978] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:15:00.978+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:15:00.978] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:15:00.978+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:15:00.978] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:15:00.979+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:15:00.979] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:15:01.225+0000] [.info] [src/director/director.go:tracef:794] [01:15:01.225] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:15:01.225+0000] [.info] [src/director/director.go:tracef:794] [01:15:01.225] because ['currentState.Up' = false] [2021-10-22T01:15:01.460+0000] [.info] [src/director/director.go:tracef:794] [01:15:01.460] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:15:01.460+0000] [.info] [src/director/director.go:tracef:794] [01:15:01.460] because ['currentState.Up' = false] [2021-10-22T01:15:01.460+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:15:01.460] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:15:01.669+0000] [.info] [src/director/director.go:tracef:794] [01:15:01.668] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:15:01.669+0000] [.info] [src/director/director.go:tracef:794] [01:15:01.669] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:15:01.684+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:01.684] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:01.716+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:15:01.716] clusterConfig unchanged [2021-10-22T01:15:01.909+0000] [.info] [src/director/director.go:tracef:794] [01:15:01.909] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:15:01.909+0000] [.info] [src/director/director.go:tracef:794] [01:15:01.909] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:15:01.910+0000] [.info] [src/director/director.go:tracef:794] [01:15:01.910] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:15:01.910+0000] [.info] [src/director/director.go:tracef:794] [01:15:01.910] because ['currentState.Up' = false] [2021-10-22T01:15:01.910+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:15:01.910] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:15:01.910+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:15:01.910] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:15:01.946+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:15:01.946] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:15:01.947+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:15:01.947] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52631 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:15:02.660+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:15:02.660] ...Finished start [pid=30891] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52631,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:15:02.682+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:02.682] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:02.802+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:15:02.802] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 142.490881ms. [2021-10-22T01:15:02.802+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:15:02.802] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52631 (local=false) [2021-10-22T01:15:02.802+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:15:02.802] Dropping existing documents from system.replset [2021-10-22T01:15:02.858+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:15:02.858] clusterConfig unchanged [2021-10-22T01:15:02.925+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:15:02.925] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:15:02.929+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:15:02.929] Inserting oplogTruncateAfterPoint document [2021-10-22T01:15:02.929+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:15:02.929] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:15:02.930+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:15:02.930] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:15:02.930+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:15:02.930] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:15:02.945+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:15:02.945] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:15:02.945+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:15:02.945] Setting the minvalid document to [{_id ObjectID("61721096523370002becc4cd")} {t -1} {ts {0 1}}] [2021-10-22T01:15:02.958+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:15:02.958] Inserted [{_id ObjectID("61721096523370002becc4cd")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:15:02.958+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:15:02.958] Inserting new replica set configuration document. [2021-10-22T01:15:02.958+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:15:02.958] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:15:02.971+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:15:02.971] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:15:02.975+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:15:02.975] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:15:02.975+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:15:02.975] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:15:02.975+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:15:02.975] Initiating stopEphemeralMongo [2021-10-22T01:15:02.975+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:15:02.975] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52631 (local=false) [2021-10-22T01:15:02.976+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:15:02.975] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52631 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:15:03.413+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:15:03.413] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.143549ms. [2021-10-22T01:15:03.413+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:15:03.413] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52631 (local=false) [2021-10-22T01:15:03.414+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:15:03.414] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.702µs. [2021-10-22T01:15:03.414+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:15:03.414] Successfully finished running stopEphemeralMongo [2021-10-22T01:15:03.414+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:15:03.414] Successfully stopped existing ephemeral mongod. [2021-10-22T01:15:03.414+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:15:03.414] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:15:03.414+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:15:03.414] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:15:03.453+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:15:03.453] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:15:03.453+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:15:03.453] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37372 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:15:03.685+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:03.685] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:03.981+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:15:03.981] clusterConfig unchanged [2021-10-22T01:15:04.621+0000] [.error] [src/action/start.go:func1:93] [103] [01:15:04.621] Error starting mongod : [01:15:04.621] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37372,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37372 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 32460 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:15:03.505+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:15:03.505+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:15:03.505+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37372,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:15:03.507+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:15:03.510+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:15:03.521+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:15:03.561+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:15:03.561+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:15:03.564+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:15:03.614+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:15:03.614+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:15:04.114+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865304:114859][32460:0x7f46aa194bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:15:04.216+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865304:216437][32460:0x7f46aa194bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:15:04.316+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865304:316278][32460:0x7f46aa194bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T01:15:04.429+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865304:429172][32460:0x7f46aa194bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:15:04.506+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865304:506843][32460:0x7f46aa194bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:15:04.572+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865304:572498][32460:0x7f46aa194bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:15:04.572+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865304:572572][32460:0x7f46aa194bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:15:04.573+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:15:04.622+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:15:04.622] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37372 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:15:04.621] Error starting mongod : [01:15:04.621] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37372,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37372 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 32460 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:15:03.505+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:15:03.505+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:15:03.505+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37372,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:15:03.507+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:15:03.510+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:15:03.521+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:15:03.561+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:15:03.561+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:15:03.564+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:15:03.614+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:15:03.614+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:15:04.114+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865304:114859][32460:0x7f46aa194bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:15:04.216+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865304:216437][32460:0x7f46aa194bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:15:04.622+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:15:04.622] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:15:04.622] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37372 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:15:04.621] Error starting mongod : [01:15:04.621] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37372,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37372 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 32460 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:15:03.505+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:15:03.505+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:15:03.505+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37372,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:15:03.507+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:15:03.510+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:15:03.521+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:15:03.561+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:15:03.561+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:15:03.564+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:15:03.614+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:15:03.614+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:15:04.114+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865304:114859][32460:0x7f46aa194bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:15:04.216+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:15:04.622+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:15:04.622] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:15:04.622+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:15:04.622] Initiating stopEphemeralMongo [2021-10-22T01:15:04.622+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:15:04.622] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:15:04.623+0000] [.error] [src/director/director.go:executePlan:962] [01:15:04.622] Failed to apply action. Result = : [01:15:04.622] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:15:04.622] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37372 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:15:04.621] Error starting mongod : [01:15:04.621] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37372,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37372 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 32460 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:15:03.505+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:15:03.505+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:15:03.505+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37372,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:15:03.507+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:15:03.510+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:15:03.521+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:15:03.561+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:15:03.561+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:15:03.564+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:15:03.614+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:15:03.614+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:15:04.114+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865304:114859][32460:0x7f46aa194bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:15:04.623+0000] [.error] [src/director/director.go:planAndExecute:567] [01:15:04.623] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:15:04.622] Failed to apply action. Result = : [01:15:04.622] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:15:04.622] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37372 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:15:04.621] Error starting mongod : [01:15:04.621] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37372,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37372 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 32460 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:15:03.505+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:15:03.505+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:15:03.505+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37372,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:15:03.507+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:15:03.510+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:15:03.521+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:15:03.561+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:15:03.561+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:15:03.564+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:15:03.614+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:15:03.614+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:15:04.114+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:15:04.623+0000] [.error] [src/director/director.go:mainLoop:399] [01:15:04.623] Failed to planAndExecute : [01:15:04.623] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:15:04.622] Failed to apply action. Result = : [01:15:04.622] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:15:04.622] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37372 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:15:04.621] Error starting mongod : [01:15:04.621] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37372,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37372 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 32460 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:15:03.505+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:15:03.505+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:15:03.505+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37372,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:15:03.507+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:15:03.510+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:15:03.521+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:15:03.561+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:15:03.561+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:15:03.564+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:15:03.614+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:15:03.614+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:15:04.114+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:15:04.685+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:04.685] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:05.241+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:15:05.241] clusterConfig unchanged [2021-10-22T01:15:05.685+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:05.685] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:06.685+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:06.685] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:07.686+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:07.686] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:08.686+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:08.686] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:09.687+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:09.687] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:10.688+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:10.688] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:11.689+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:11.689] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:12.690+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:12.690] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:13.410+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:15:13.410] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.430430218s [2021-10-22T01:15:13.687+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:13.687] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:14.688+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:14.688] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:15.382+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:15:15.382] clusterConfig unchanged [2021-10-22T01:15:15.689+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:15.689] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:16.690+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:16.690] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:17.690+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:17.690] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:18.691+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:18.691] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:19.691+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:19.691] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:20.692+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:20.692] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:21.693+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:21.693] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:22.694+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:22.694] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:23.695+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:23.695] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:24.701+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:24.701] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:25.494+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:15:25.494] clusterConfig unchanged [2021-10-22T01:15:25.696+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:25.696] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:26.697+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:26.697] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:27.697+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:27.697] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:28.698+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:28.698] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:29.698+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:29.698] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:30.699+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:30.699] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:31.700+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:31.700] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:32.700+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:32.700] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:33.701+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:33.701] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:34.702+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:34.702] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:35.606+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:15:35.606] clusterConfig unchanged [2021-10-22T01:15:35.702+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:35.702] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:36.708+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:36.708] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:37.703+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:37.703] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:38.704+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:38.704] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:39.705+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:39.705] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:40.705+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:40.705] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:41.315+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:15:41.315] memory:rss=52580352 alloc=9658016 totalalloc=4709632408 mallocs=60271229 frees=60197585 heapinuse=14827520 heapobjects=73644 gcpausetotalns=1474963854 gcpauselastns=103831 gcnum=745 utimens=86690820000 stimens=29371575000 minflt=234776 majflt=405 [2021-10-22T01:15:41.706+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:41.706] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:42.707+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:42.707] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:43.711+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:43.711] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:44.707+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:44.707] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:45.714+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:45.714] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:45.738+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:15:45.738] clusterConfig unchanged [2021-10-22T01:15:46.708+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:46.708] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:47.708+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:47.708] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:48.709+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:48.709] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:49.710+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:49.710] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:50.710+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:50.710] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:51.711+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:51.711] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:52.711+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:52.711] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:53.712+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:53.712] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:54.712+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:54.712] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:55.713+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:55.713] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:55.862+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:15:55.862] clusterConfig unchanged [2021-10-22T01:15:56.713+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:56.713] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:57.714+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:57.714] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:58.714+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:58.714] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:15:59.714+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:15:59.714] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:00.715+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:00.715] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:01.715+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:01.715] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:02.715+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:02.715] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:03.716+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:03.716] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:04.719+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:04.719] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:04.785+0000] [.info] [src/director/director.go:computePlan:280] [01:16:04.785] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:16:04.785+0000] [.info] [src/director/director.go:tracef:794] [01:16:04.785] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:16:04.785+0000] [.info] [src/director/director.go:tracef:794] [01:16:04.785] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:16:04.785+0000] [.info] [src/director/director.go:tracef:794] [01:16:04.785] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:16:04.785+0000] [.info] [src/director/director.go:tracef:794] [01:16:04.785] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:16:04.786+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:16:04.786] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:16:04.937+0000] [.info] [src/director/director.go:tracef:794] [01:16:04.937] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:16:04.937+0000] [.info] [src/director/director.go:tracef:794] [01:16:04.937] because [2021-10-22T01:16:04.937+0000] [.info] [src/director/director.go:tracef:794] [01:16:04.937] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:16:04.937+0000] [.info] [src/director/director.go:tracef:794] [01:16:04.937] because ['currentState.CanBounceStop' = true] [2021-10-22T01:16:05.095+0000] [.info] [src/director/director.go:tracef:794] [01:16:05.095] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:16:05.095+0000] [.info] [src/director/director.go:tracef:794] [01:16:05.095] because [2021-10-22T01:16:05.095+0000] [.info] [src/director/director.go:tracef:794] [01:16:05.095] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:16:05.095+0000] [.info] [src/director/director.go:tracef:794] [01:16:05.095] because ['currentState.Up' = false] [2021-10-22T01:16:05.095+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:16:05.095] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:16:05.095+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:16:05.095] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:16:05.174+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:16:05.174] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:16:05.182+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:16:05.182] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:16:05.183+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:16:05.183] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:16:05.184+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:16:05.184] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:16:05.185+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:16:05.185] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:16:05.185+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:16:05.185] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:16:05.185+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:16:05.185] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:16:05.186+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:16:05.186] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:16:05.420+0000] [.info] [src/director/director.go:tracef:794] [01:16:05.420] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:16:05.420+0000] [.info] [src/director/director.go:tracef:794] [01:16:05.420] because ['currentState.Up' = false] [2021-10-22T01:16:05.636+0000] [.info] [src/director/director.go:tracef:794] [01:16:05.636] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:16:05.636+0000] [.info] [src/director/director.go:tracef:794] [01:16:05.636] because ['currentState.Up' = false] [2021-10-22T01:16:05.636+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:16:05.636] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:16:05.722+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:05.722] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:05.842+0000] [.info] [src/director/director.go:tracef:794] [01:16:05.842] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:16:05.842+0000] [.info] [src/director/director.go:tracef:794] [01:16:05.842] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:16:06.043+0000] [.info] [src/director/director.go:tracef:794] [01:16:06.043] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:16:06.043+0000] [.info] [src/director/director.go:tracef:794] [01:16:06.043] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:16:06.043+0000] [.info] [src/director/director.go:tracef:794] [01:16:06.043] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:16:06.043+0000] [.info] [src/director/director.go:tracef:794] [01:16:06.043] because ['currentState.Up' = false] [2021-10-22T01:16:06.043+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:16:06.043] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:16:06.043+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:16:06.043] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:16:06.075+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:16:06.075] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:16:06.076+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:16:06.076] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35052 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:16:06.080+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:16:06.080] clusterConfig unchanged [2021-10-22T01:16:06.720+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:06.720] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:06.806+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:16:06.806] ...Finished start [pid=51530] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35052,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:16:06.926+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:16:06.926] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 118.926684ms. [2021-10-22T01:16:06.926+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:16:06.926] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:35052 (local=false) [2021-10-22T01:16:06.926+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:16:06.926] Dropping existing documents from system.replset [2021-10-22T01:16:07.077+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:16:07.077] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:16:07.081+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:16:07.081] Inserting oplogTruncateAfterPoint document [2021-10-22T01:16:07.081+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:16:07.081] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:16:07.082+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:16:07.082] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:16:07.082+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:16:07.082] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:16:07.094+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:16:07.094] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:16:07.094+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:16:07.094] Setting the minvalid document to [{_id ObjectID("617210d7523370002becc4db")} {t -1} {ts {0 1}}] [2021-10-22T01:16:07.106+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:16:07.106] Inserted [{_id ObjectID("617210d7523370002becc4db")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:16:07.106+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:16:07.106] Inserting new replica set configuration document. [2021-10-22T01:16:07.106+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:16:07.106] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:16:07.119+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:16:07.119] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:16:07.153+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:16:07.153] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:16:07.153+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:16:07.153] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:16:07.153+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:16:07.153] Initiating stopEphemeralMongo [2021-10-22T01:16:07.153+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:16:07.153] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:35052 (local=false) [2021-10-22T01:16:07.153+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:16:07.153] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:35052 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:16:07.202+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:16:07.202] clusterConfig unchanged [2021-10-22T01:16:07.497+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:16:07.497] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.564854ms. [2021-10-22T01:16:07.497+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:16:07.497] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:35052 (local=false) [2021-10-22T01:16:07.497+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:16:07.497] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 38.496µs. [2021-10-22T01:16:07.497+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:16:07.497] Successfully finished running stopEphemeralMongo [2021-10-22T01:16:07.497+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:16:07.497] Successfully stopped existing ephemeral mongod. [2021-10-22T01:16:07.497+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:16:07.497] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:16:07.497+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:16:07.497] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:16:07.534+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:16:07.534] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:16:07.535+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:16:07.534] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39649 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:16:07.724+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:07.724] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:08.326+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:16:08.326] clusterConfig unchanged [2021-10-22T01:16:08.689+0000] [.error] [src/action/start.go:func1:93] [103] [01:16:08.689] Error starting mongod : [01:16:08.689] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39649 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:16:07.564Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 53118 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:16:07.584+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:16:07.584+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:16:07.584+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:16:07.585+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:16:07.589+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:16:07.590+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:16:07.611+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:16:07.611+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:16:07.612+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:16:07.659+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:16:07.659+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:16:08.188+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865368:188787][53118:0x7f0010a35bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:16:08.288+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865368:288591][53118:0x7f0010a35bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:16:08.390+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865368:390399][53118:0x7f0010a35bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35456 to 2/256"}} {"t":{"$date":"2021-10-22T01:16:08.501+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865368:501268][53118:0x7f0010a35bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:16:08.578+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865368:578920][53118:0x7f0010a35bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:16:08.638+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865368:638553][53118:0x7f0010a35bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:16:08.638+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865368:638623][53118:0x7f0010a35bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:16:08.689+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:16:08.689] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39649 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:16:08.689] Error starting mongod : [01:16:08.689] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39649 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:16:07.564Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 53118 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:16:07.584+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:16:07.584+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:16:07.584+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:16:07.585+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:16:07.589+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:16:07.590+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:16:07.611+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:16:07.611+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:16:07.612+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:16:07.659+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:16:07.659+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:16:08.188+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865368:188787][53118:0x7f0010a35bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:16:08.288+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:16:08.689+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:16:08.689] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:16:08.689] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39649 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:16:08.689] Error starting mongod : [01:16:08.689] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39649 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:16:07.564Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 53118 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:16:07.584+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:16:07.584+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:16:07.584+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:16:07.585+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:16:07.589+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:16:07.590+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:16:07.611+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:16:07.611+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:16:07.612+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:16:07.659+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:16:07.659+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:16:08.188+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865368:188787][53118:0x7f0010a35bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:16:08.689+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:16:08.689] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:16:08.689+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:16:08.689] Initiating stopEphemeralMongo [2021-10-22T01:16:08.689+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:16:08.689] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:16:08.690+0000] [.error] [src/director/director.go:executePlan:962] [01:16:08.690] Failed to apply action. Result = : [01:16:08.689] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:16:08.689] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39649 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:16:08.689] Error starting mongod : [01:16:08.689] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39649 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:16:07.564Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 53118 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:16:07.584+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:16:07.584+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:16:07.584+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:16:07.585+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:16:07.589+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:16:07.590+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:16:07.611+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:16:07.611+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:16:07.612+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:16:07.659+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:16:07.659+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:16:08.188+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:16:08.690+0000] [.error] [src/director/director.go:planAndExecute:567] [01:16:08.690] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:16:08.690] Failed to apply action. Result = : [01:16:08.689] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:16:08.689] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39649 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:16:08.689] Error starting mongod : [01:16:08.689] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39649 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:16:07.564Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 53118 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:16:07.584+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:16:07.584+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:16:07.584+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:16:07.585+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:16:07.589+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:16:07.590+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:16:07.611+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:16:07.611+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:16:07.612+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:16:07.659+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:16:07.659+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:16:08.690+0000] [.error] [src/director/director.go:mainLoop:399] [01:16:08.690] Failed to planAndExecute : [01:16:08.690] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:16:08.690] Failed to apply action. Result = : [01:16:08.689] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:16:08.689] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39649 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:16:08.689] Error starting mongod : [01:16:08.689] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39649 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:16:07.564Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 53118 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:16:07.584+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:16:07.584+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:16:07.584+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:16:07.585+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:16:07.589+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:16:07.590+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:16:07.611+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:16:07.611+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:16:07.612+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:16:07.659+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:16:07.659+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:16:08.724+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:08.724] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:09.585+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:16:09.585] clusterConfig unchanged [2021-10-22T01:16:09.724+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:09.724] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:10.731+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:10.731] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:11.732+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:11.732] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:12.733+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:12.732] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:13.731+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:13.731] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:14.734+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:14.734] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:15.734+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:15.734] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:16.734+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:16.734] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:17.461+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:16:17.461] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.274979589s [2021-10-22T01:16:17.732+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:17.732] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:18.559+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:16:18.559] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:16:18.561+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:18.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:16:18.561+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [01:16:18.561] Could not get version : [01:16:18.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:16:18.561+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [01:16:18.561] Error getting version for log lines. Skipping : [01:16:18.561] Could not get version : [01:16:18.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:16:18.732+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:18.732] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:19.720+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:16:19.720] clusterConfig unchanged [2021-10-22T01:16:19.732+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:19.732] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:20.733+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:20.733] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:21.734+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:21.734] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:22.734+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:22.734] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:23.735+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:23.735] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:24.736+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:24.736] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:25.736+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:25.736] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:26.736+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:26.736] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:27.737+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:27.737] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:28.738+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:28.738] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:29.738+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:29.738] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:29.824+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:16:29.824] clusterConfig unchanged [2021-10-22T01:16:30.739+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:30.739] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:31.740+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:31.740] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:32.740+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:32.740] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:33.745+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:33.745] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:34.740+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:34.740] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:35.741+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:35.741] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:36.741+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:36.741] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:37.741+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:37.741] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:38.743+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:38.743] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:39.743+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:39.743] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:39.951+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:16:39.951] clusterConfig unchanged [2021-10-22T01:16:40.744+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:40.744] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:41.316+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:16:41.316] memory:rss=52408320 alloc=11144256 totalalloc=4798250784 mallocs=61384885 frees=61295156 heapinuse=15212544 heapobjects=89729 gcpausetotalns=1477209985 gcpauselastns=109570 gcnum=758 utimens=88487619000 stimens=30026499000 minflt=239663 majflt=405 [2021-10-22T01:16:41.747+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:41.747] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:42.749+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:42.749] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:43.745+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:43.745] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:44.745+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:44.745] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:45.747+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:45.747] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:46.747+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:46.747] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:47.748+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:47.748] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:48.748+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:48.748] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:49.748+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:49.748] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:50.054+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:16:50.054] clusterConfig unchanged [2021-10-22T01:16:50.748+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:50.748] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:51.749+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:51.749] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:52.750+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:52.750] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:53.750+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:53.750] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:54.757+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:54.757] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:55.751+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:55.751] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:56.751+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:56.751] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:57.752+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:57.752] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:58.753+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:58.753] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:16:59.753+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:16:59.753] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:00.156+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:17:00.156] clusterConfig unchanged [2021-10-22T01:17:00.754+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:00.754] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:01.755+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:01.755] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:02.759+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:02.759] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:03.757+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:03.757] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:04.759+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:04.759] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:05.764+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:05.764] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:06.759+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:06.759] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:07.759+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:07.759] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:08.762+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:08.762] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:08.857+0000] [.info] [src/director/director.go:computePlan:280] [01:17:08.857] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:17:08.858+0000] [.info] [src/director/director.go:tracef:794] [01:17:08.858] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:17:08.858+0000] [.info] [src/director/director.go:tracef:794] [01:17:08.858] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:17:08.858+0000] [.info] [src/director/director.go:tracef:794] [01:17:08.858] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:17:08.858+0000] [.info] [src/director/director.go:tracef:794] [01:17:08.858] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:17:08.858+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:17:08.858] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:17:09.030+0000] [.info] [src/director/director.go:tracef:794] [01:17:09.030] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:17:09.030+0000] [.info] [src/director/director.go:tracef:794] [01:17:09.030] because [2021-10-22T01:17:09.031+0000] [.info] [src/director/director.go:tracef:794] [01:17:09.031] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:17:09.031+0000] [.info] [src/director/director.go:tracef:794] [01:17:09.031] because ['currentState.CanBounceStop' = true] [2021-10-22T01:17:09.198+0000] [.info] [src/director/director.go:tracef:794] [01:17:09.198] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:17:09.198+0000] [.info] [src/director/director.go:tracef:794] [01:17:09.198] because [2021-10-22T01:17:09.198+0000] [.info] [src/director/director.go:tracef:794] [01:17:09.198] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:17:09.198+0000] [.info] [src/director/director.go:tracef:794] [01:17:09.198] because ['currentState.Up' = false] [2021-10-22T01:17:09.198+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:17:09.198] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:17:09.198+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:17:09.198] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:17:09.284+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:17:09.284] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:17:09.292+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:17:09.292] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:17:09.292+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:17:09.292] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:17:09.293+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:17:09.293] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:17:09.294+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:17:09.294] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:17:09.294+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:17:09.294] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:17:09.294+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:17:09.294] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:17:09.295+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:17:09.295] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:17:09.506+0000] [.info] [src/director/director.go:tracef:794] [01:17:09.506] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:17:09.506+0000] [.info] [src/director/director.go:tracef:794] [01:17:09.506] because ['currentState.Up' = false] [2021-10-22T01:17:09.724+0000] [.info] [src/director/director.go:tracef:794] [01:17:09.724] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:17:09.724+0000] [.info] [src/director/director.go:tracef:794] [01:17:09.724] because ['currentState.Up' = false] [2021-10-22T01:17:09.724+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:17:09.724] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:17:09.776+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:09.776] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:09.935+0000] [.info] [src/director/director.go:tracef:794] [01:17:09.935] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:17:09.935+0000] [.info] [src/director/director.go:tracef:794] [01:17:09.935] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:17:10.163+0000] [.info] [src/director/director.go:tracef:794] [01:17:10.163] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:17:10.163+0000] [.info] [src/director/director.go:tracef:794] [01:17:10.163] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:17:10.163+0000] [.info] [src/director/director.go:tracef:794] [01:17:10.163] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:17:10.163+0000] [.info] [src/director/director.go:tracef:794] [01:17:10.163] because ['currentState.Up' = false] [2021-10-22T01:17:10.163+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:17:10.163] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:17:10.163+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:17:10.163] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:17:10.198+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:17:10.198] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:17:10.198+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:17:10.198] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49144 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:17:10.372+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:17:10.372] clusterConfig unchanged [2021-10-22T01:17:10.786+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:10.786] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:10.951+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:17:10.950] ...Finished start [pid=72302] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49144,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:17:11.074+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:17:11.074] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 122.928135ms. [2021-10-22T01:17:11.074+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:17:11.074] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49144 (local=false) [2021-10-22T01:17:11.074+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:17:11.074] Dropping existing documents from system.replset [2021-10-22T01:17:11.211+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:17:11.211] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:17:11.214+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:17:11.214] Inserting oplogTruncateAfterPoint document [2021-10-22T01:17:11.214+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:17:11.214] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:17:11.215+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:17:11.215] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:17:11.215+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:17:11.215] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:17:11.229+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:17:11.229] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:17:11.229+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:17:11.229] Setting the minvalid document to [{_id ObjectID("61721117523370002becc4e9")} {t -1} {ts {0 1}}] [2021-10-22T01:17:11.242+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:17:11.242] Inserted [{_id ObjectID("61721117523370002becc4e9")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:17:11.242+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:17:11.242] Inserting new replica set configuration document. [2021-10-22T01:17:11.242+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:17:11.242] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:17:11.255+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:17:11.255] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:17:11.259+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:17:11.259] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:17:11.259+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:17:11.259] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:17:11.259+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:17:11.259] Initiating stopEphemeralMongo [2021-10-22T01:17:11.259+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:17:11.259] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49144 (local=false) [2021-10-22T01:17:11.259+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:17:11.259] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49144 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:17:11.508+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:17:11.508] clusterConfig unchanged [2021-10-22T01:17:11.737+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:17:11.737] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.06838ms. [2021-10-22T01:17:11.737+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:17:11.737] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49144 (local=false) [2021-10-22T01:17:11.737+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:17:11.737] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.164µs. [2021-10-22T01:17:11.737+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:17:11.737] Successfully finished running stopEphemeralMongo [2021-10-22T01:17:11.737+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:17:11.737] Successfully stopped existing ephemeral mongod. [2021-10-22T01:17:11.737+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:17:11.737] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:17:11.737+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:17:11.737] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:17:11.772+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:17:11.772] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:17:11.773+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:17:11.773] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36246 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:17:11.791+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:11.791] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:12.652+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:17:12.652] clusterConfig unchanged [2021-10-22T01:17:12.783+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:12.783] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:12.934+0000] [.error] [src/action/start.go:func1:93] [103] [01:17:12.934] Error starting mongod : [01:17:12.934] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36246,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36246 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:17:11.812Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 73907 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:17:11.833+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:17:11.833+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:17:11.833+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36246,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:17:11.835+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:17:11.837+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:17:11.838+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:17:11.903+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:17:11.903+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:17:11.904+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:17:11.949+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:17:11.949+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:17:12.432+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865432:432901][73907:0x7f3683180bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:17:12.536+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865432:536376][73907:0x7f3683180bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:17:12.636+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865432:636683][73907:0x7f3683180bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:17:12.749+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865432:749319][73907:0x7f3683180bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:17:12.825+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865432:825355][73907:0x7f3683180bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:17:12.885+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865432:885104][73907:0x7f3683180bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:17:12.885+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865432:885181][73907:0x7f3683180bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:17:12.935+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:17:12.934] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36246 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:17:12.934] Error starting mongod : [01:17:12.934] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36246,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36246 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:17:11.812Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 73907 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:17:11.833+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:17:11.833+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:17:11.833+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36246,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:17:11.835+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:17:11.837+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:17:11.838+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:17:11.903+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:17:11.903+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:17:11.904+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:17:11.949+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:17:11.949+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:17:12.432+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865432:432901][73907:0x7f3683180bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:17:12.536+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:17:12.935+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:17:12.935] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:17:12.934] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36246 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:17:12.934] Error starting mongod : [01:17:12.934] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36246,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36246 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:17:11.812Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 73907 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:17:11.833+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:17:11.833+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:17:11.833+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36246,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:17:11.835+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:17:11.837+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:17:11.838+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:17:11.903+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:17:11.903+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:17:11.904+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:17:11.949+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:17:11.949+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:17:12.432+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865432:432901][73907:0x7f3683180bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:17:12.935+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:17:12.935] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:17:12.935+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:17:12.935] Initiating stopEphemeralMongo [2021-10-22T01:17:12.935+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:17:12.935] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:17:12.936+0000] [.error] [src/director/director.go:executePlan:962] [01:17:12.935] Failed to apply action. Result = : [01:17:12.935] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:17:12.934] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36246 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:17:12.934] Error starting mongod : [01:17:12.934] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36246,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36246 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:17:11.812Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 73907 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:17:11.833+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:17:11.833+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:17:11.833+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36246,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:17:11.835+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:17:11.837+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:17:11.838+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:17:11.903+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:17:11.903+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:17:11.904+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:17:11.949+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:17:11.949+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:17:12.432+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:17:12.936+0000] [.error] [src/director/director.go:planAndExecute:567] [01:17:12.936] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:17:12.935] Failed to apply action. Result = : [01:17:12.935] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:17:12.934] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36246 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:17:12.934] Error starting mongod : [01:17:12.934] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36246,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36246 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:17:11.812Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 73907 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:17:11.833+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:17:11.833+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:17:11.833+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36246,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:17:11.835+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:17:11.837+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:17:11.838+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:17:11.903+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:17:11.903+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:17:11.904+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:17:11.949+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:17:11.949+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:17:12.936+0000] [.error] [src/director/director.go:mainLoop:399] [01:17:12.936] Failed to planAndExecute : [01:17:12.936] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:17:12.935] Failed to apply action. Result = : [01:17:12.935] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:17:12.934] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36246 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:17:12.934] Error starting mongod : [01:17:12.934] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36246,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36246 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:17:11.812Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 73907 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:17:11.833+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:17:11.833+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:17:11.833+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36246,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:17:11.835+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:17:11.837+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:17:11.838+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:17:11.903+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:17:11.903+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:17:11.904+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:17:11.949+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:17:11.949+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:17:13.781+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:13.781] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:13.891+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:17:13.891] clusterConfig unchanged [2021-10-22T01:17:14.782+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:14.782] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:15.783+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:15.783] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:16.783+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:16.783] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:17.783+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:17.783] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:18.784+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:18.784] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:19.787+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:19.787] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:20.783+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:20.783] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:21.749+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:17:21.749] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.453814976s [2021-10-22T01:17:21.782+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:21.782] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:22.784+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:22.784] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:23.784+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:23.784] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:24.023+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:17:24.023] clusterConfig unchanged [2021-10-22T01:17:24.784+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:24.784] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:25.785+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:25.785] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:26.786+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:26.786] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:27.561+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:17:27.561] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:17:27.563+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:17:27.563] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:17:27.787+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:27.787] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:28.789+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:28.789] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:29.788+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:29.788] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:30.789+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:30.789] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:31.790+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:31.790] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:32.790+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:32.790] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:33.790+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:33.790] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:34.184+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:17:34.184] clusterConfig unchanged [2021-10-22T01:17:34.791+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:34.791] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:35.792+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [01:17:35.792] lsblk cache invalidated, running lsblk command [2021-10-22T01:17:35.794+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:35.794] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:36.792+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:36.792] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:37.791+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:37.791] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:38.792+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:38.792] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:39.792+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:39.792] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:40.793+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:40.793] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:41.316+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:17:41.316] memory:rss=52187136 alloc=12543776 totalalloc=4885294456 mallocs=62488546 frees=62384944 heapinuse=16228352 heapobjects=103602 gcpausetotalns=1478424499 gcpauselastns=93884 gcnum=771 utimens=90335842000 stimens=30718199000 minflt=244231 majflt=405 [2021-10-22T01:17:41.794+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:41.794] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:42.796+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:42.796] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:43.796+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:43.796] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:44.234+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [01:17:44.234] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36246 (local=false) because it is not up [2021-10-22T01:17:44.316+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:17:44.316] clusterConfig unchanged [2021-10-22T01:17:44.796+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:44.796] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:45.797+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:45.797] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:46.801+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:46.801] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:47.799+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:47.799] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:48.800+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:48.800] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:49.806+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:49.806] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:50.800+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:50.800] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:51.800+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:51.800] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:52.801+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:52.801] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:53.801+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:53.801] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:54.408+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:17:54.408] clusterConfig unchanged [2021-10-22T01:17:54.801+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:54.801] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:55.802+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:55.802] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:56.803+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:56.803] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:57.803+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:57.803] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:58.803+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:58.803] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:17:59.809+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:17:59.809] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:00.804+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:00.804] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:01.804+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:01.804] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:02.805+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:02.805] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:03.806+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:03.806] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:04.510+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:18:04.510] clusterConfig unchanged [2021-10-22T01:18:04.806+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:04.806] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:05.806+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:05.806] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:06.807+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:06.807] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:07.807+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:07.807] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:08.807+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:08.807] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:09.807+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:09.807] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:10.813+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:10.813] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:11.807+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:11.807] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:12.808+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:12.808] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:13.108+0000] [.info] [src/director/director.go:computePlan:280] [01:18:13.108] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:18:13.108+0000] [.info] [src/director/director.go:tracef:794] [01:18:13.108] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:18:13.108+0000] [.info] [src/director/director.go:tracef:794] [01:18:13.108] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:18:13.108+0000] [.info] [src/director/director.go:tracef:794] [01:18:13.108] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:18:13.108+0000] [.info] [src/director/director.go:tracef:794] [01:18:13.108] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:18:13.108+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:18:13.108] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:18:13.262+0000] [.info] [src/director/director.go:tracef:794] [01:18:13.262] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:18:13.262+0000] [.info] [src/director/director.go:tracef:794] [01:18:13.262] because [2021-10-22T01:18:13.262+0000] [.info] [src/director/director.go:tracef:794] [01:18:13.262] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:18:13.262+0000] [.info] [src/director/director.go:tracef:794] [01:18:13.262] because ['currentState.CanBounceStop' = true] [2021-10-22T01:18:13.410+0000] [.info] [src/director/director.go:tracef:794] [01:18:13.410] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:18:13.410+0000] [.info] [src/director/director.go:tracef:794] [01:18:13.410] because [2021-10-22T01:18:13.410+0000] [.info] [src/director/director.go:tracef:794] [01:18:13.410] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:18:13.410+0000] [.info] [src/director/director.go:tracef:794] [01:18:13.410] because ['currentState.Up' = false] [2021-10-22T01:18:13.410+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:18:13.410] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:18:13.410+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:18:13.410] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:18:13.490+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:18:13.490] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:18:13.491+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:18:13.490] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:18:13.498+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:18:13.498] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:18:13.501+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:18:13.501] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:18:13.503+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:18:13.503] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:18:13.503+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:18:13.503] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:18:13.505+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:18:13.505] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:18:13.507+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:18:13.507] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:18:13.720+0000] [.info] [src/director/director.go:tracef:794] [01:18:13.720] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:18:13.720+0000] [.info] [src/director/director.go:tracef:794] [01:18:13.720] because ['currentState.Up' = false] [2021-10-22T01:18:13.819+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:13.819] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:13.940+0000] [.info] [src/director/director.go:tracef:794] [01:18:13.940] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:18:13.940+0000] [.info] [src/director/director.go:tracef:794] [01:18:13.940] because ['currentState.Up' = false] [2021-10-22T01:18:13.940+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:18:13.940] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:18:14.162+0000] [.info] [src/director/director.go:tracef:794] [01:18:14.162] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:18:14.162+0000] [.info] [src/director/director.go:tracef:794] [01:18:14.162] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:18:14.361+0000] [.info] [src/director/director.go:tracef:794] [01:18:14.361] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:18:14.361+0000] [.info] [src/director/director.go:tracef:794] [01:18:14.361] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:18:14.361+0000] [.info] [src/director/director.go:tracef:794] [01:18:14.361] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:18:14.361+0000] [.info] [src/director/director.go:tracef:794] [01:18:14.361] because ['currentState.Up' = false] [2021-10-22T01:18:14.361+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:18:14.361] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:18:14.361+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:18:14.361] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:18:14.394+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:18:14.394] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:18:14.394+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:18:14.394] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36016 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:18:14.721+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:18:14.721] clusterConfig unchanged [2021-10-22T01:18:14.813+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:14.813] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:15.111+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:18:15.111] ...Finished start [pid=19094] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36016,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:18:15.244+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:18:15.244] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 132.700563ms. [2021-10-22T01:18:15.244+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:18:15.244] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36016 (local=false) [2021-10-22T01:18:15.244+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:18:15.244] Dropping existing documents from system.replset [2021-10-22T01:18:15.367+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:18:15.367] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:18:15.370+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:18:15.370] Inserting oplogTruncateAfterPoint document [2021-10-22T01:18:15.370+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:18:15.370] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:18:15.372+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:18:15.372] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:18:15.372+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:18:15.372] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:18:15.384+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:18:15.384] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:18:15.384+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:18:15.384] Setting the minvalid document to [{_id ObjectID("61721157523370002becc4f7")} {t -1} {ts {0 1}}] [2021-10-22T01:18:15.397+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:18:15.397] Inserted [{_id ObjectID("61721157523370002becc4f7")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:18:15.397+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:18:15.397] Inserting new replica set configuration document. [2021-10-22T01:18:15.398+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:18:15.397] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:18:15.412+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:18:15.412] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:18:15.415+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:18:15.415] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:18:15.415+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:18:15.415] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:18:15.415+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:18:15.415] Initiating stopEphemeralMongo [2021-10-22T01:18:15.415+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:18:15.415] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36016 (local=false) [2021-10-22T01:18:15.415+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:18:15.415] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36016 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:18:15.755+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:18:15.755] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.11532ms. [2021-10-22T01:18:15.755+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:18:15.755] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36016 (local=false) [2021-10-22T01:18:15.755+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:18:15.755] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.94µs. [2021-10-22T01:18:15.755+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:18:15.755] Successfully finished running stopEphemeralMongo [2021-10-22T01:18:15.755+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:18:15.755] Successfully stopped existing ephemeral mongod. [2021-10-22T01:18:15.755+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:18:15.755] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:18:15.755+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:18:15.755] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:18:15.787+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:18:15.787] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:18:15.788+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:18:15.788] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58060 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:18:15.812+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:15.812] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:15.834+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:18:15.834] clusterConfig unchanged [2021-10-22T01:18:16.814+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:16.814] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:16.937+0000] [.error] [src/action/start.go:func1:93] [103] [01:18:16.937] Error starting mongod : [01:18:16.937] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58060,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58060 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:18:15.822Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 20508 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:18:15.840+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:18:15.840+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:18:15.840+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58060,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:18:15.841+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:18:15.843+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:18:15.843+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:18:15.858+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:18:15.858+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:18:15.859+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:18:15.904+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:18:15.904+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:18:16.443+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865496:443100][20508:0x7f2b9d846bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:18:16.545+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865496:545435][20508:0x7f2b9d846bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:18:16.645+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865496:645930][20508:0x7f2b9d846bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:18:16.757+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865496:757808][20508:0x7f2b9d846bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:18:16.829+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865496:828992][20508:0x7f2b9d846bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:18:16.887+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865496:887318][20508:0x7f2b9d846bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:18:16.887+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865496:887388][20508:0x7f2b9d846bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:18:16.937+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:18:16.937] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:58060 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:18:16.937] Error starting mongod : [01:18:16.937] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58060,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58060 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:18:15.822Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 20508 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:18:15.840+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:18:15.840+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:18:15.840+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58060,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:18:15.841+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:18:15.843+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:18:15.843+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:18:15.858+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:18:15.858+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:18:15.859+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:18:15.904+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:18:15.904+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:18:16.443+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865496:443100][20508:0x7f2b9d846bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:18:16.545+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:18:16.937+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:18:16.937] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:18:16.937] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:58060 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:18:16.937] Error starting mongod : [01:18:16.937] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58060,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58060 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:18:15.822Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 20508 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:18:15.840+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:18:15.840+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:18:15.840+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58060,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:18:15.841+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:18:15.843+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:18:15.843+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:18:15.858+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:18:15.858+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:18:15.859+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:18:15.904+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:18:15.904+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:18:16.443+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865496:443100][20508:0x7f2b9d846bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:18:16.938+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:18:16.938] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:18:16.938+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:18:16.938] Initiating stopEphemeralMongo [2021-10-22T01:18:16.938+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:18:16.938] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:18:16.938+0000] [.error] [src/director/director.go:executePlan:962] [01:18:16.938] Failed to apply action. Result = : [01:18:16.937] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:18:16.937] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:58060 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:18:16.937] Error starting mongod : [01:18:16.937] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58060,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58060 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:18:15.822Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 20508 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:18:15.840+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:18:15.840+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:18:15.840+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58060,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:18:15.841+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:18:15.843+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:18:15.843+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:18:15.858+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:18:15.858+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:18:15.859+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:18:15.904+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:18:15.904+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:18:16.443+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:18:16.938+0000] [.error] [src/director/director.go:planAndExecute:567] [01:18:16.938] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:18:16.938] Failed to apply action. Result = : [01:18:16.937] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:18:16.937] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:58060 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:18:16.937] Error starting mongod : [01:18:16.937] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58060,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58060 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:18:15.822Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 20508 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:18:15.840+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:18:15.840+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:18:15.840+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58060,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:18:15.841+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:18:15.843+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:18:15.843+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:18:15.858+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:18:15.858+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:18:15.859+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:18:15.904+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:18:15.904+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:18:16.938+0000] [.error] [src/director/director.go:mainLoop:399] [01:18:16.938] Failed to planAndExecute : [01:18:16.938] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:18:16.938] Failed to apply action. Result = : [01:18:16.937] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:18:16.937] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:58060 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:18:16.937] Error starting mongod : [01:18:16.937] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58060,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58060 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:18:15.822Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 20508 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:18:15.840+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:18:15.840+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:18:15.840+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58060,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:18:15.841+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:18:15.843+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:18:15.843+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:18:15.858+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:18:15.858+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:18:15.859+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:18:15.904+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:18:15.904+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:18:16.976+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:18:16.976] clusterConfig unchanged [2021-10-22T01:18:17.814+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:17.814] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:18.226+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:18:18.226] clusterConfig unchanged [2021-10-22T01:18:18.814+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:18.814] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:19.815+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:19.815] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:20.815+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:20.815] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:21.816+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:21.816] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:22.816+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:22.816] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:23.816+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:23.816] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:24.817+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:24.817] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:25.794+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:18:25.794] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.287625771s [2021-10-22T01:18:25.818+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:25.818] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:26.830+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:26.830] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:27.819+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:27.819] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:28.328+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:18:28.328] clusterConfig unchanged [2021-10-22T01:18:28.818+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:28.818] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:29.821+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:29.821] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:30.821+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:30.820] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:31.823+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:31.823] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:32.822+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:32.822] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:33.823+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:33.823] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:34.825+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:34.825] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:35.825+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:35.825] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:36.825+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:36.825] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:37.827+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:37.827] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:38.431+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:18:38.431] clusterConfig unchanged [2021-10-22T01:18:38.833+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:38.833] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:39.826+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:39.826] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:40.826+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:40.826] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:41.317+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:18:41.317] memory:rss=51621888 alloc=8286648 totalalloc=4973591048 mallocs=63602150 frees=63542880 heapinuse=14475264 heapobjects=59270 gcpausetotalns=1483941778 gcpauselastns=2231282 gcnum=785 utimens=92326533000 stimens=31609258000 minflt=249489 majflt=405 [2021-10-22T01:18:41.828+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:41.828] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:42.829+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:42.829] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:43.829+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:43.829] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:44.832+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:44.832] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:45.831+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:45.831] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:46.831+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:46.831] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:47.838+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:47.838] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:48.562+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:18:48.562] clusterConfig unchanged [2021-10-22T01:18:48.832+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:48.832] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:49.832+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:49.832] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:50.833+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:50.833] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:51.834+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:51.834] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:52.835+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:52.835] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:53.836+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:53.836] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:54.837+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:54.837] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:55.838+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:55.838] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:56.839+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:56.839] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:57.839+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:57.839] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:58.674+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:18:58.674] clusterConfig unchanged [2021-10-22T01:18:58.839+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:58.839] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:18:59.840+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:18:59.840] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:00.841+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:00.841] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:01.846+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:01.846] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:02.843+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:02.843] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:03.843+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:03.843] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:04.844+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:04.844] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:05.844+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:05.844] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:06.860+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:06.860] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:07.845+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:07.845] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:08.806+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:19:08.806] clusterConfig unchanged [2021-10-22T01:19:08.853+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:08.853] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:09.847+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:09.847] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:10.847+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:10.847] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:11.848+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:11.848] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:12.848+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:12.848] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:13.848+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:13.848] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:14.849+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:14.849] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:15.849+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:15.849] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:16.850+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:16.850] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:17.102+0000] [.info] [src/director/director.go:computePlan:280] [01:19:17.102] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:19:17.102+0000] [.info] [src/director/director.go:tracef:794] [01:19:17.102] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:19:17.102+0000] [.info] [src/director/director.go:tracef:794] [01:19:17.102] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:19:17.102+0000] [.info] [src/director/director.go:tracef:794] [01:19:17.102] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:19:17.102+0000] [.info] [src/director/director.go:tracef:794] [01:19:17.102] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:19:17.102+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:19:17.102] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:19:17.258+0000] [.info] [src/director/director.go:tracef:794] [01:19:17.258] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:19:17.258+0000] [.info] [src/director/director.go:tracef:794] [01:19:17.258] because [2021-10-22T01:19:17.258+0000] [.info] [src/director/director.go:tracef:794] [01:19:17.258] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:19:17.258+0000] [.info] [src/director/director.go:tracef:794] [01:19:17.258] because ['currentState.CanBounceStop' = true] [2021-10-22T01:19:17.410+0000] [.info] [src/director/director.go:tracef:794] [01:19:17.410] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:19:17.410+0000] [.info] [src/director/director.go:tracef:794] [01:19:17.410] because [2021-10-22T01:19:17.411+0000] [.info] [src/director/director.go:tracef:794] [01:19:17.411] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:19:17.411+0000] [.info] [src/director/director.go:tracef:794] [01:19:17.411] because ['currentState.Up' = false] [2021-10-22T01:19:17.411+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:19:17.411] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:19:17.411+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:19:17.411] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:19:17.500+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:19:17.500] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:19:17.508+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:19:17.508] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:19:17.509+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:19:17.509] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:19:17.511+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:19:17.511] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:19:17.511+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:19:17.511] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:19:17.511+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:19:17.511] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:19:17.512+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:19:17.512] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:19:17.730+0000] [.info] [src/director/director.go:tracef:794] [01:19:17.730] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:19:17.730+0000] [.info] [src/director/director.go:tracef:794] [01:19:17.730] because ['currentState.Up' = false] [2021-10-22T01:19:17.861+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:17.861] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:17.945+0000] [.info] [src/director/director.go:tracef:794] [01:19:17.945] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:19:17.946+0000] [.info] [src/director/director.go:tracef:794] [01:19:17.946] because ['currentState.Up' = false] [2021-10-22T01:19:17.946+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:19:17.946] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:19:18.167+0000] [.info] [src/director/director.go:tracef:794] [01:19:18.167] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:19:18.167+0000] [.info] [src/director/director.go:tracef:794] [01:19:18.167] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:19:18.372+0000] [.info] [src/director/director.go:tracef:794] [01:19:18.371] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:19:18.372+0000] [.info] [src/director/director.go:tracef:794] [01:19:18.372] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:19:18.372+0000] [.info] [src/director/director.go:tracef:794] [01:19:18.372] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:19:18.372+0000] [.info] [src/director/director.go:tracef:794] [01:19:18.372] because ['currentState.Up' = false] [2021-10-22T01:19:18.372+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:19:18.372] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:19:18.372+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:19:18.372] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:19:18.406+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:19:18.406] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:19:18.406+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:19:18.406] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56584 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:19:18.872+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:18.872] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:19.027+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:19:19.027] clusterConfig unchanged [2021-10-22T01:19:19.131+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:19:19.131] ...Finished start [pid=40348] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56584,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:19:19.264+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:19:19.264] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 131.9885ms. [2021-10-22T01:19:19.264+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:19:19.264] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:56584 (local=false) [2021-10-22T01:19:19.264+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:19:19.264] Dropping existing documents from system.replset [2021-10-22T01:19:19.387+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:19:19.387] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:19:19.391+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:19:19.391] Inserting oplogTruncateAfterPoint document [2021-10-22T01:19:19.391+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:19:19.391] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:19:19.392+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:19:19.392] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:19:19.392+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:19:19.392] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:19:19.405+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:19:19.405] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:19:19.405+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:19:19.405] Setting the minvalid document to [{_id ObjectID("61721197523370002becc505")} {t -1} {ts {0 1}}] [2021-10-22T01:19:19.417+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:19:19.417] Inserted [{_id ObjectID("61721197523370002becc505")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:19:19.417+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:19:19.417] Inserting new replica set configuration document. [2021-10-22T01:19:19.417+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:19:19.417] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:19:19.431+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:19:19.431] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:19:19.434+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:19:19.434] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:19:19.434+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:19:19.434] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:19:19.434+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:19:19.434] Initiating stopEphemeralMongo [2021-10-22T01:19:19.434+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:19:19.434] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:56584 (local=false) [2021-10-22T01:19:19.434+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:19:19.434] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:56584 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:19:19.873+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:19.873] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:19.979+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:19:19.979] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.997939ms. [2021-10-22T01:19:19.979+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:19:19.979] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:56584 (local=false) [2021-10-22T01:19:19.979+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:19:19.979] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 65.686µs. [2021-10-22T01:19:19.979+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:19:19.979] Successfully finished running stopEphemeralMongo [2021-10-22T01:19:19.979+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:19:19.979] Successfully stopped existing ephemeral mongod. [2021-10-22T01:19:19.979+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:19:19.979] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:19:19.979+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:19:19.979] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:19:20.027+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:19:20.027] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:19:20.028+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:19:20.028] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53114 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:19:20.150+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:19:20.150] clusterConfig unchanged [2021-10-22T01:19:20.881+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:20.881] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:21.271+0000] [.error] [src/action/start.go:func1:93] [103] [01:19:21.271] Error starting mongod : [01:19:21.271] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53114 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:19:20.060Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 42050 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:19:20.092+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:19:20.092+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:19:20.092+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:19:20.094+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:19:20.096+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:19:20.096+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:19:20.133+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:19:20.133+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:19:20.148+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:19:20.193+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:19:20.193+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:19:20.743+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865560:743080][42050:0x7f19af36ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:19:20.861+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865560:861276][42050:0x7f19af36ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:19:20.972+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865560:972016][42050:0x7f19af36ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:19:21.086+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865561:86569][42050:0x7f19af36ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:19:21.158+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865561:158809][42050:0x7f19af36ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:19:21.218+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865561:218887][42050:0x7f19af36ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:19:21.218+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865561:218974][42050:0x7f19af36ebc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:19:21.271+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:19:21.271] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53114 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:19:21.271] Error starting mongod : [01:19:21.271] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53114 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:19:20.060Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 42050 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:19:20.092+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:19:20.092+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:19:20.092+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:19:20.094+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:19:20.096+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:19:20.096+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:19:20.133+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:19:20.133+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:19:20.148+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:19:20.193+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:19:20.193+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:19:20.743+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865560:743080][42050:0x7f19af36ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:19:20.861+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:19:21.272+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:19:21.272] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:19:21.271] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53114 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:19:21.271] Error starting mongod : [01:19:21.271] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53114 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:19:20.060Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 42050 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:19:20.092+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:19:20.092+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:19:20.092+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:19:20.094+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:19:20.096+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:19:20.096+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:19:20.133+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:19:20.133+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:19:20.148+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:19:20.193+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:19:20.193+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:19:20.743+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865560:743080][42050:0x7f19af36ebc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:19:21.272+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:19:21.272] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:19:21.272+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:19:21.272] Initiating stopEphemeralMongo [2021-10-22T01:19:21.272+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:19:21.272] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:19:21.272+0000] [.error] [src/director/director.go:executePlan:962] [01:19:21.272] Failed to apply action. Result = : [01:19:21.272] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:19:21.271] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53114 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:19:21.271] Error starting mongod : [01:19:21.271] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53114 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:19:20.060Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 42050 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:19:20.092+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:19:20.092+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:19:20.092+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:19:20.094+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:19:20.096+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:19:20.096+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:19:20.133+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:19:20.133+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:19:20.148+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:19:20.193+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:19:20.193+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:19:20.743+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:19:21.272+0000] [.error] [src/director/director.go:planAndExecute:567] [01:19:21.272] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:19:21.272] Failed to apply action. Result = : [01:19:21.272] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:19:21.271] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53114 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:19:21.271] Error starting mongod : [01:19:21.271] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53114 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:19:20.060Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 42050 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:19:20.092+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:19:20.092+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:19:20.092+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:19:20.094+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:19:20.096+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:19:20.096+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:19:20.133+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:19:20.133+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:19:20.148+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:19:20.193+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:19:20.193+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:19:21.273+0000] [.error] [src/director/director.go:mainLoop:399] [01:19:21.273] Failed to planAndExecute : [01:19:21.272] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:19:21.272] Failed to apply action. Result = : [01:19:21.272] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:19:21.271] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53114 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:19:21.271] Error starting mongod : [01:19:21.271] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53114 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:19:20.060Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 42050 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:19:20.092+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:19:20.092+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:19:20.092+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53114,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:19:20.094+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:19:20.096+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:19:20.096+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:19:20.133+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:19:20.133+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:19:20.148+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:19:20.193+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:19:20.193+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:19:21.274+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:19:21.273] clusterConfig unchanged [2021-10-22T01:19:21.881+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:21.881] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:22.522+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:19:22.522] clusterConfig unchanged [2021-10-22T01:19:22.882+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:22.882] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:23.883+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:23.883] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:24.883+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:24.883] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:25.883+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:25.883] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:26.883+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:26.883] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:27.884+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:27.884] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:28.885+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:28.885] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:29.886+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:29.886] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:30.022+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:19:30.022] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.509618217s [2021-10-22T01:19:30.884+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:30.884] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:31.884+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:31.884] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:32.644+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:19:32.644] clusterConfig unchanged [2021-10-22T01:19:32.884+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:32.884] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:33.885+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:33.885] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:34.886+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:34.886] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:35.886+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:35.886] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:36.886+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:36.886] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:37.887+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:37.887] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:38.888+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:38.888] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:39.892+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:39.892] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:40.889+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:40.889] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:41.317+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:19:41.317] memory:rss=52330496 alloc=7981256 totalalloc=5061059424 mallocs=64709174 frees=64654071 heapinuse=14770176 heapobjects=55103 gcpausetotalns=1486930280 gcpauselastns=98822 gcnum=798 utimens=94249336000 stimens=32395311000 minflt=253877 majflt=405 [2021-10-22T01:19:41.890+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:41.890] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:42.765+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:19:42.765] clusterConfig unchanged [2021-10-22T01:19:42.890+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:42.890] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:43.890+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:43.890] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:44.891+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:44.891] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:45.892+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:45.892] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:46.892+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:46.892] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:47.893+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:47.893] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:48.908+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:48.908] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:49.893+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:49.893] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:50.894+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:50.894] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:51.894+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:51.894] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:52.888+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:19:52.888] clusterConfig unchanged [2021-10-22T01:19:52.895+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:52.895] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:53.895+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:53.895] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:54.896+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:54.896] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:55.896+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:55.896] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:56.897+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:56.897] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:57.898+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:57.898] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:58.904+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:58.904] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:19:59.899+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:19:59.899] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:00.900+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:00.900] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:01.901+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:01.901] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:02.900+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:02.900] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:03.019+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:20:03.019] clusterConfig unchanged [2021-10-22T01:20:03.901+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:03.901] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:04.901+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:04.901] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:05.902+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:05.902] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:06.902+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:06.902] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:07.902+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:07.902] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:08.902+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:08.902] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:09.908+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:09.908] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:10.903+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:10.903] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:11.905+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:11.905] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:12.905+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:12.905] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:13.140+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:20:13.140] clusterConfig unchanged [2021-10-22T01:20:13.905+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:13.905] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:14.905+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:14.905] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:15.906+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:15.906] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:16.906+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:16.906] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:17.907+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:17.907] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:18.907+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:18.907] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:19.907+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:19.907] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:20.908+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:20.908] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:21.439+0000] [.info] [src/director/director.go:computePlan:280] [01:20:21.439] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:20:21.439+0000] [.info] [src/director/director.go:tracef:794] [01:20:21.439] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:20:21.439+0000] [.info] [src/director/director.go:tracef:794] [01:20:21.439] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:20:21.439+0000] [.info] [src/director/director.go:tracef:794] [01:20:21.439] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:20:21.439+0000] [.info] [src/director/director.go:tracef:794] [01:20:21.439] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:20:21.439+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:20:21.439] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:20:21.620+0000] [.info] [src/director/director.go:tracef:794] [01:20:21.620] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:20:21.620+0000] [.info] [src/director/director.go:tracef:794] [01:20:21.620] because [2021-10-22T01:20:21.620+0000] [.info] [src/director/director.go:tracef:794] [01:20:21.620] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:20:21.621+0000] [.info] [src/director/director.go:tracef:794] [01:20:21.621] because ['currentState.CanBounceStop' = true] [2021-10-22T01:20:21.772+0000] [.info] [src/director/director.go:tracef:794] [01:20:21.772] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:20:21.772+0000] [.info] [src/director/director.go:tracef:794] [01:20:21.772] because [2021-10-22T01:20:21.772+0000] [.info] [src/director/director.go:tracef:794] [01:20:21.772] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:20:21.772+0000] [.info] [src/director/director.go:tracef:794] [01:20:21.772] because ['currentState.Up' = false] [2021-10-22T01:20:21.772+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:20:21.772] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:20:21.772+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:20:21.772] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:20:21.855+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:20:21.855] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:20:21.863+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:20:21.863] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:20:21.864+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:20:21.864] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:20:21.866+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:20:21.866] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:20:21.866+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:20:21.866] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:20:21.867+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:20:21.867] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:20:21.870+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:20:21.870] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:20:21.917+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:21.917] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:22.084+0000] [.info] [src/director/director.go:tracef:794] [01:20:22.084] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:20:22.084+0000] [.info] [src/director/director.go:tracef:794] [01:20:22.084] because ['currentState.Up' = false] [2021-10-22T01:20:22.288+0000] [.info] [src/director/director.go:tracef:794] [01:20:22.288] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:20:22.288+0000] [.info] [src/director/director.go:tracef:794] [01:20:22.288] because ['currentState.Up' = false] [2021-10-22T01:20:22.288+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:20:22.288] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:20:22.504+0000] [.info] [src/director/director.go:tracef:794] [01:20:22.504] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:20:22.505+0000] [.info] [src/director/director.go:tracef:794] [01:20:22.505] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:20:22.716+0000] [.info] [src/director/director.go:tracef:794] [01:20:22.716] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:20:22.716+0000] [.info] [src/director/director.go:tracef:794] [01:20:22.716] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:20:22.716+0000] [.info] [src/director/director.go:tracef:794] [01:20:22.716] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:20:22.716+0000] [.info] [src/director/director.go:tracef:794] [01:20:22.716] because ['currentState.Up' = false] [2021-10-22T01:20:22.716+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:20:22.716] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:20:22.716+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:20:22.716] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:20:22.748+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:20:22.748] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:20:22.749+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:20:22.749] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52417 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:20:22.917+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:22.917] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:23.361+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:20:23.361] clusterConfig unchanged [2021-10-22T01:20:23.472+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:20:23.471] ...Finished start [pid=61074] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52417,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:20:23.593+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:20:23.593] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 121.058947ms. [2021-10-22T01:20:23.593+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:20:23.593] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52417 (local=false) [2021-10-22T01:20:23.593+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:20:23.593] Dropping existing documents from system.replset [2021-10-22T01:20:23.711+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:20:23.711] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:20:23.715+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:20:23.715] Inserting oplogTruncateAfterPoint document [2021-10-22T01:20:23.715+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:20:23.715] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:20:23.716+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:20:23.716] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:20:23.716+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:20:23.716] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:20:23.731+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:20:23.731] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:20:23.731+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:20:23.731] Setting the minvalid document to [{_id ObjectID("617211d7523370002becc513")} {t -1} {ts {0 1}}] [2021-10-22T01:20:23.743+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:20:23.743] Inserted [{_id ObjectID("617211d7523370002becc513")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:20:23.744+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:20:23.744] Inserting new replica set configuration document. [2021-10-22T01:20:23.744+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:20:23.744] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:20:23.756+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:20:23.756] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:20:23.760+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:20:23.760] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:20:23.760+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:20:23.760] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:20:23.760+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:20:23.760] Initiating stopEphemeralMongo [2021-10-22T01:20:23.760+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:20:23.760] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52417 (local=false) [2021-10-22T01:20:23.760+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:20:23.760] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52417 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:20:23.918+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:23.918] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:24.149+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:20:24.149] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.176918ms. [2021-10-22T01:20:24.149+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:20:24.149] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52417 (local=false) [2021-10-22T01:20:24.149+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:20:24.149] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.511µs. [2021-10-22T01:20:24.150+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:20:24.150] Successfully finished running stopEphemeralMongo [2021-10-22T01:20:24.150+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:20:24.150] Successfully stopped existing ephemeral mongod. [2021-10-22T01:20:24.150+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:20:24.150] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:20:24.150+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:20:24.150] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:20:24.185+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:20:24.185] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:20:24.185+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:20:24.185] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39948 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:20:24.503+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:20:24.503] clusterConfig unchanged [2021-10-22T01:20:24.918+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:24.918] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:25.394+0000] [.error] [src/action/start.go:func1:93] [103] [01:20:25.394] Error starting mongod : [01:20:25.393] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39948,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39948 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:20:24.217Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 62620 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:20:24.238+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:20:24.238+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:20:24.238+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39948,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:20:24.240+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:20:24.241+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:20:24.242+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:20:24.286+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:20:24.286+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:20:24.287+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:20:24.341+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:20:24.341+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:20:24.892+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865624:892016][62620:0x7f37b472ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:20:24.996+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865624:996975][62620:0x7f37b472ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:20:25.097+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865625:97034][62620:0x7f37b472ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:20:25.212+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865625:212225][62620:0x7f37b472ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:20:25.285+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865625:285601][62620:0x7f37b472ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:20:25.343+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865625:343309][62620:0x7f37b472ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:20:25.343+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865625:343393][62620:0x7f37b472ebc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:20:25.394+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:20:25.394] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39948 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:20:25.394] Error starting mongod : [01:20:25.393] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39948,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39948 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:20:24.217Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 62620 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:20:24.238+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:20:24.238+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:20:24.238+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39948,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:20:24.240+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:20:24.241+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:20:24.242+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:20:24.286+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:20:24.286+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:20:24.287+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:20:24.341+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:20:24.341+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:20:24.892+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865624:892016][62620:0x7f37b472ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:20:24.996+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:20:25.394+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:20:25.394] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:20:25.394] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39948 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:20:25.394] Error starting mongod : [01:20:25.393] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39948,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39948 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:20:24.217Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 62620 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:20:24.238+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:20:24.238+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:20:24.238+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39948,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:20:24.240+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:20:24.241+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:20:24.242+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:20:24.286+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:20:24.286+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:20:24.287+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:20:24.341+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:20:24.341+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:20:24.892+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865624:892016][62620:0x7f37b472ebc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:20:25.394+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:20:25.394] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:20:25.394+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:20:25.394] Initiating stopEphemeralMongo [2021-10-22T01:20:25.394+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:20:25.394] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:20:25.395+0000] [.error] [src/director/director.go:executePlan:962] [01:20:25.395] Failed to apply action. Result = : [01:20:25.394] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:20:25.394] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39948 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:20:25.394] Error starting mongod : [01:20:25.393] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39948,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39948 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:20:24.217Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 62620 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:20:24.238+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:20:24.238+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:20:24.238+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39948,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:20:24.240+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:20:24.241+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:20:24.242+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:20:24.286+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:20:24.286+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:20:24.287+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:20:24.341+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:20:24.341+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:20:24.892+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:20:25.395+0000] [.error] [src/director/director.go:planAndExecute:567] [01:20:25.395] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:20:25.395] Failed to apply action. Result = : [01:20:25.394] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:20:25.394] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39948 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:20:25.394] Error starting mongod : [01:20:25.393] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39948,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39948 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:20:24.217Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 62620 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:20:24.238+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:20:24.238+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:20:24.238+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39948,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:20:24.240+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:20:24.241+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:20:24.242+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:20:24.286+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:20:24.286+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:20:24.287+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:20:24.341+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:20:24.341+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:20:25.395+0000] [.error] [src/director/director.go:mainLoop:399] [01:20:25.395] Failed to planAndExecute : [01:20:25.395] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:20:25.395] Failed to apply action. Result = : [01:20:25.394] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:20:25.394] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39948 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:20:25.394] Error starting mongod : [01:20:25.393] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39948,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39948 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:20:24.217Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 62620 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:20:24.238+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:20:24.238+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:20:24.238+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39948,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:20:24.240+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:20:24.241+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:20:24.242+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:20:24.286+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:20:24.286+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:20:24.287+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:20:24.341+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:20:24.341+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:20:25.793+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:20:25.793] clusterConfig unchanged [2021-10-22T01:20:25.919+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:25.919] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:26.919+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:26.919] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:27.917+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:27.917] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:28.921+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:28.921] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:29.921+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:29.921] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:30.920+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:30.920] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:31.921+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:31.921] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:32.922+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:32.922] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:33.921+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:33.921] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:34.260+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:20:34.260] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.390843757s [2021-10-22T01:20:34.920+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:34.920] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:35.920+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:35.920] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:35.954+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:20:35.954] clusterConfig unchanged [2021-10-22T01:20:36.921+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:36.921] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:37.922+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:37.922] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:38.923+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:38.923] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:39.923+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:39.923] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:40.923+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:40.923] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:41.318+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:20:41.318] memory:rss=52858880 alloc=11966208 totalalloc=5148498968 mallocs=65819926 frees=65723441 heapinuse=16252928 heapobjects=96485 gcpausetotalns=1488236620 gcpauselastns=101936 gcnum=811 utimens=96048788000 stimens=33076598000 minflt=259046 majflt=405 [2021-10-22T01:20:41.924+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:41.924] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:42.925+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:42.925] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:43.925+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:43.925] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:44.242+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [01:20:44.242] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39948 (local=false) because it is not up [2021-10-22T01:20:44.926+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:44.926] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:45.926+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:45.926] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:46.096+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:20:46.096] clusterConfig unchanged [2021-10-22T01:20:46.927+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:46.927] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:47.928+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:47.928] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:48.928+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:48.928] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:49.928+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:49.928] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:50.928+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:50.928] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:51.929+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:51.929] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:52.930+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:52.930] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:53.930+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:53.930] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:54.931+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:54.930] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:55.931+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:55.931] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:56.237+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:20:56.237] clusterConfig unchanged [2021-10-22T01:20:56.931+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:56.931] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:57.931+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:57.931] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:58.932+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:58.932] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:20:59.932+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:20:59.932] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:00.933+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:00.933] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:01.934+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:01.934] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:02.935+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:02.935] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:03.941+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:03.941] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:04.936+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:04.936] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:05.937+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:05.937] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:06.339+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:21:06.339] clusterConfig unchanged [2021-10-22T01:21:06.937+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:06.937] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:07.938+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:07.938] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:08.939+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:08.939] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:09.939+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:09.939] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:10.940+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:10.940] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:11.941+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:11.941] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:12.941+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:12.941] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:13.942+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:13.942] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:14.943+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:14.943] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:15.950+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:15.950] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:16.462+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:21:16.462] clusterConfig unchanged [2021-10-22T01:21:16.944+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:16.944] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:17.946+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:17.946] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:18.559+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:21:18.559] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:21:18.561+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:18.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:21:18.561+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [01:21:18.561] Could not get version : [01:21:18.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:21:18.561+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [01:21:18.561] Error getting version for log lines. Skipping : [01:21:18.561] Could not get version : [01:21:18.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:21:18.946+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:18.946] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:19.946+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:19.946] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:20.946+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:20.946] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:21.947+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:21.946] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:22.947+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:22.947] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:23.947+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:23.947] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:24.955+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:24.955] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:25.563+0000] [.info] [src/director/director.go:computePlan:280] [01:21:25.563] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:21:25.563+0000] [.info] [src/director/director.go:tracef:794] [01:21:25.563] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:21:25.563+0000] [.info] [src/director/director.go:tracef:794] [01:21:25.563] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:21:25.563+0000] [.info] [src/director/director.go:tracef:794] [01:21:25.563] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:21:25.563+0000] [.info] [src/director/director.go:tracef:794] [01:21:25.563] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:21:25.564+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:21:25.564] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:21:25.711+0000] [.info] [src/director/director.go:tracef:794] [01:21:25.711] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:21:25.711+0000] [.info] [src/director/director.go:tracef:794] [01:21:25.711] because [2021-10-22T01:21:25.712+0000] [.info] [src/director/director.go:tracef:794] [01:21:25.712] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:21:25.712+0000] [.info] [src/director/director.go:tracef:794] [01:21:25.712] because ['currentState.CanBounceStop' = true] [2021-10-22T01:21:25.866+0000] [.info] [src/director/director.go:tracef:794] [01:21:25.866] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:21:25.866+0000] [.info] [src/director/director.go:tracef:794] [01:21:25.866] because [2021-10-22T01:21:25.866+0000] [.info] [src/director/director.go:tracef:794] [01:21:25.866] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:21:25.866+0000] [.info] [src/director/director.go:tracef:794] [01:21:25.866] because ['currentState.Up' = false] [2021-10-22T01:21:25.867+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:21:25.867] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:21:25.867+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:21:25.867] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:21:25.949+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:25.949] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:25.953+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:21:25.953] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:21:25.961+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:21:25.961] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:21:25.962+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:21:25.962] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:21:25.962+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:21:25.962] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:21:25.963+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:21:25.963] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:21:25.963+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:21:25.963] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:21:25.963+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:21:25.963] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:21:25.964+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:21:25.964] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:21:26.279+0000] [.info] [src/director/director.go:tracef:794] [01:21:26.279] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:21:26.280+0000] [.info] [src/director/director.go:tracef:794] [01:21:26.280] because ['currentState.Up' = false] [2021-10-22T01:21:26.560+0000] [.info] [src/director/director.go:tracef:794] [01:21:26.560] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:21:26.560+0000] [.info] [src/director/director.go:tracef:794] [01:21:26.560] because ['currentState.Up' = false] [2021-10-22T01:21:26.560+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:21:26.560] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:21:26.687+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:21:26.687] clusterConfig unchanged [2021-10-22T01:21:26.779+0000] [.info] [src/director/director.go:tracef:794] [01:21:26.779] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:21:26.780+0000] [.info] [src/director/director.go:tracef:794] [01:21:26.780] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:21:26.972+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:26.972] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:27.012+0000] [.info] [src/director/director.go:tracef:794] [01:21:27.012] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:21:27.012+0000] [.info] [src/director/director.go:tracef:794] [01:21:27.012] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:21:27.012+0000] [.info] [src/director/director.go:tracef:794] [01:21:27.012] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:21:27.012+0000] [.info] [src/director/director.go:tracef:794] [01:21:27.012] because ['currentState.Up' = false] [2021-10-22T01:21:27.012+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:21:27.012] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:21:27.012+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:21:27.012] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:21:27.047+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:21:27.047] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:21:27.047+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:21:27.047] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42480 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:21:27.804+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:21:27.804] clusterConfig unchanged [2021-10-22T01:21:27.836+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:21:27.836] ...Finished start [pid=7188] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42480,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:21:27.947+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:21:27.947] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 111.111264ms. [2021-10-22T01:21:27.948+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:21:27.948] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42480 (local=false) [2021-10-22T01:21:27.948+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:21:27.948] Dropping existing documents from system.replset [2021-10-22T01:21:27.965+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:27.965] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:28.133+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:21:28.133] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:21:28.137+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:21:28.137] Inserting oplogTruncateAfterPoint document [2021-10-22T01:21:28.137+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:21:28.137] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:21:28.138+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:21:28.138] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:21:28.138+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:21:28.138] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:21:28.151+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:21:28.151] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:21:28.152+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:21:28.152] Setting the minvalid document to [{_id ObjectID("61721218523370002becc521")} {t -1} {ts {0 1}}] [2021-10-22T01:21:28.165+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:21:28.165] Inserted [{_id ObjectID("61721218523370002becc521")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:21:28.165+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:21:28.165] Inserting new replica set configuration document. [2021-10-22T01:21:28.165+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:21:28.165] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:21:28.177+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:21:28.177] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:21:28.201+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:21:28.201] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:21:28.201+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:21:28.201] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:21:28.201+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:21:28.201] Initiating stopEphemeralMongo [2021-10-22T01:21:28.201+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:21:28.201] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42480 (local=false) [2021-10-22T01:21:28.201+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:21:28.201] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42480 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:21:28.599+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:21:28.599] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.686349ms. [2021-10-22T01:21:28.599+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:21:28.599] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42480 (local=false) [2021-10-22T01:21:28.599+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:21:28.599] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.757µs. [2021-10-22T01:21:28.599+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:21:28.599] Successfully finished running stopEphemeralMongo [2021-10-22T01:21:28.599+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:21:28.599] Successfully stopped existing ephemeral mongod. [2021-10-22T01:21:28.599+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:21:28.599] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:21:28.599+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:21:28.599] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:21:28.635+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:21:28.635] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:21:28.636+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:21:28.636] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46398 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:21:28.913+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:21:28.913] clusterConfig unchanged [2021-10-22T01:21:28.971+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:28.971] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:29.802+0000] [.error] [src/action/start.go:func1:93] [103] [01:21:29.802] Error starting mongod : [01:21:29.802] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46398,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46398 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 8670 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:21:28.700+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:21:28.700+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:21:28.700+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46398,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:21:28.702+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:21:28.704+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:21:28.704+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:21:28.747+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:21:28.747+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:21:28.758+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:21:28.810+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:21:28.810+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:21:29.301+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865689:301120][8670:0x7efcac99abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:21:29.401+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865689:401674][8670:0x7efcac99abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:21:29.512+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865689:512364][8670:0x7efcac99abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35456 to 2/256"}} {"t":{"$date":"2021-10-22T01:21:29.625+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865689:625548][8670:0x7efcac99abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:21:29.695+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865689:695516][8670:0x7efcac99abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:21:29.752+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865689:752344][8670:0x7efcac99abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:21:29.752+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865689:752415][8670:0x7efcac99abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:21:29.754+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:21:29.802+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:21:29.802] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46398 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:21:29.802] Error starting mongod : [01:21:29.802] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46398,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46398 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 8670 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:21:28.700+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:21:28.700+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:21:28.700+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46398,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:21:28.702+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:21:28.704+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:21:28.704+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:21:28.747+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:21:28.747+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:21:28.758+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:21:28.810+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:21:28.810+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:21:29.301+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865689:301120][8670:0x7efcac99abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:21:29.401+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865689:401674][8670:0x7efcac99abc0], txn-recover...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:21:29.802+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:21:29.802] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:21:29.802] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46398 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:21:29.802] Error starting mongod : [01:21:29.802] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46398,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46398 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 8670 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:21:28.700+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:21:28.700+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:21:28.700+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46398,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:21:28.702+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:21:28.704+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:21:28.704+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:21:28.747+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:21:28.747+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:21:28.758+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:21:28.810+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:21:28.810+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:21:29.301+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865689:301120][8670:0x7efcac99abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:21:29.401+00:00"},"s":"I", "c":"STORAGE", "id":22430, ... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:21:29.802+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:21:29.802] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:21:29.803+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:21:29.802] Initiating stopEphemeralMongo [2021-10-22T01:21:29.803+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:21:29.803] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:21:29.803+0000] [.error] [src/director/director.go:executePlan:962] [01:21:29.803] Failed to apply action. Result = : [01:21:29.802] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:21:29.802] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46398 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:21:29.802] Error starting mongod : [01:21:29.802] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46398,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46398 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 8670 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:21:28.700+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:21:28.700+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:21:28.700+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46398,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:21:28.702+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:21:28.704+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:21:28.704+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:21:28.747+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:21:28.747+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:21:28.758+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:21:28.810+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:21:28.810+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:21:29.301+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865689:301120][8670:0x7efcac99abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:21:29.803+0000] [.error] [src/director/director.go:planAndExecute:567] [01:21:29.803] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:21:29.803] Failed to apply action. Result = : [01:21:29.802] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:21:29.802] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46398 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:21:29.802] Error starting mongod : [01:21:29.802] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46398,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46398 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 8670 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:21:28.700+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:21:28.700+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:21:28.700+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46398,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:21:28.702+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:21:28.704+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:21:28.704+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:21:28.747+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:21:28.747+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:21:28.758+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:21:28.810+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:21:28.810+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:21:29.301+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger m... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:21:29.803+0000] [.error] [src/director/director.go:mainLoop:399] [01:21:29.803] Failed to planAndExecute : [01:21:29.803] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:21:29.803] Failed to apply action. Result = : [01:21:29.802] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:21:29.802] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46398 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:21:29.802] Error starting mongod : [01:21:29.802] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46398,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46398 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 8670 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:21:28.700+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:21:28.700+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:21:28.700+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46398,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:21:28.702+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:21:28.704+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:21:28.704+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:21:28.747+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:21:28.747+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:21:28.758+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:21:28.810+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:21:28.810+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:21:29.301+00:00"},"s":"I",... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:21:29.967+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:29.967] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:30.158+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:21:30.158] clusterConfig unchanged [2021-10-22T01:21:30.970+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:30.970] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:31.970+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:31.970] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:32.971+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:32.971] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:33.974+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:33.974] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:34.971+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:34.971] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:35.973+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:35.973] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:36.974+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:36.974] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:37.974+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:37.974] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:38.870+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:21:38.870] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.905824257s [2021-10-22T01:21:38.972+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:38.972] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:39.973+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:39.973] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:40.249+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:21:40.249] clusterConfig unchanged [2021-10-22T01:21:40.974+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:40.974] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:41.320+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:21:41.320] memory:rss=53161984 alloc=12186800 totalalloc=5237986696 mallocs=66939722 frees=66839173 heapinuse=16080896 heapobjects=100549 gcpausetotalns=1492596877 gcpauselastns=89364 gcnum=824 utimens=97863392000 stimens=33763376000 minflt=263680 majflt=405 [2021-10-22T01:21:41.975+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:41.975] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:42.981+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:42.981] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:43.975+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:43.975] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:44.976+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:44.976] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:45.976+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:45.976] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:46.977+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:46.977] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:47.978+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:47.978] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:48.993+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:48.993] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:49.978+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:49.978] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:50.341+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:21:50.341] clusterConfig unchanged [2021-10-22T01:21:50.979+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:50.979] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:51.979+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:51.979] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:52.980+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:52.980] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:53.987+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:53.987] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:54.981+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:54.981] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:55.982+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:55.982] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:56.983+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:56.983] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:57.983+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:57.983] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:58.984+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:58.984] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:21:59.984+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:21:59.984] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:00.433+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:22:00.433] clusterConfig unchanged [2021-10-22T01:22:00.985+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:00.985] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:01.986+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:01.986] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:02.986+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:02.986] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:03.987+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:03.987] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:04.989+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:04.989] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:05.989+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:05.989] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:06.989+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:06.989] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:07.990+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:07.990] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:08.990+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:08.990] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:09.991+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:09.991] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:10.551+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:22:10.551] clusterConfig unchanged [2021-10-22T01:22:10.992+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:10.992] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:11.992+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:11.992] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:12.994+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:12.994] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:13.994+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:13.994] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:14.994+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:14.994] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:16.004+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:16.004] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:16.995+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:16.995] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:17.996+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:17.996] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:18.997+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:18.997] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:19.997+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:19.997] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:20.689+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:22:20.689] clusterConfig unchanged [2021-10-22T01:22:20.998+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:20.998] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:21.999+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:21.999] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:23.000+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:23.000] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:24.003+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:24.003] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:25.001+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:25.001] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:26.002+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:26.002] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:27.004+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:27.004] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:27.561+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:22:27.561] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:22:27.563+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:22:27.563] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:22:28.003+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:28.003] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:29.003+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:29.003] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:29.968+0000] [.info] [src/director/director.go:computePlan:280] [01:22:29.968] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:22:29.968+0000] [.info] [src/director/director.go:tracef:794] [01:22:29.968] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:22:29.968+0000] [.info] [src/director/director.go:tracef:794] [01:22:29.968] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:22:29.968+0000] [.info] [src/director/director.go:tracef:794] [01:22:29.968] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:22:29.968+0000] [.info] [src/director/director.go:tracef:794] [01:22:29.968] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:22:29.968+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:22:29.968] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:22:30.004+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:30.004] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:30.137+0000] [.info] [src/director/director.go:tracef:794] [01:22:30.137] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:22:30.137+0000] [.info] [src/director/director.go:tracef:794] [01:22:30.137] because [2021-10-22T01:22:30.137+0000] [.info] [src/director/director.go:tracef:794] [01:22:30.137] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:22:30.137+0000] [.info] [src/director/director.go:tracef:794] [01:22:30.137] because ['currentState.CanBounceStop' = true] [2021-10-22T01:22:30.284+0000] [.info] [src/director/director.go:tracef:794] [01:22:30.284] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:22:30.284+0000] [.info] [src/director/director.go:tracef:794] [01:22:30.284] because [2021-10-22T01:22:30.284+0000] [.info] [src/director/director.go:tracef:794] [01:22:30.284] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:22:30.284+0000] [.info] [src/director/director.go:tracef:794] [01:22:30.284] because ['currentState.Up' = false] [2021-10-22T01:22:30.284+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:22:30.284] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:22:30.284+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:22:30.284] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:22:30.372+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:22:30.372] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:22:30.380+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:22:30.380] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:22:30.381+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:22:30.381] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:22:30.382+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:22:30.382] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:22:30.383+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:22:30.383] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:22:30.383+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:22:30.383] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:22:30.383+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:22:30.383] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:22:30.384+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:22:30.384] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:22:30.591+0000] [.info] [src/director/director.go:tracef:794] [01:22:30.591] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:22:30.591+0000] [.info] [src/director/director.go:tracef:794] [01:22:30.591] because ['currentState.Up' = false] [2021-10-22T01:22:30.811+0000] [.info] [src/director/director.go:tracef:794] [01:22:30.811] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:22:30.811+0000] [.info] [src/director/director.go:tracef:794] [01:22:30.811] because ['currentState.Up' = false] [2021-10-22T01:22:30.811+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:22:30.811] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:22:30.895+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:22:30.895] clusterConfig unchanged [2021-10-22T01:22:31.007+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:31.007] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:31.025+0000] [.info] [src/director/director.go:tracef:794] [01:22:31.025] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:22:31.025+0000] [.info] [src/director/director.go:tracef:794] [01:22:31.025] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:22:31.233+0000] [.info] [src/director/director.go:tracef:794] [01:22:31.233] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:22:31.233+0000] [.info] [src/director/director.go:tracef:794] [01:22:31.233] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:22:31.233+0000] [.info] [src/director/director.go:tracef:794] [01:22:31.233] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:22:31.233+0000] [.info] [src/director/director.go:tracef:794] [01:22:31.233] because ['currentState.Up' = false] [2021-10-22T01:22:31.233+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:22:31.233] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:22:31.233+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:22:31.233] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:22:31.268+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:22:31.268] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:22:31.269+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:22:31.269] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58498 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:22:32.007+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:22:32.007] clusterConfig unchanged [2021-10-22T01:22:32.010+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:32.010] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:32.021+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:22:32.021] ...Finished start [pid=28680] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58498,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:22:32.215+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:22:32.215] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 194.129827ms. [2021-10-22T01:22:32.215+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:22:32.215] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:58498 (local=false) [2021-10-22T01:22:32.215+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:22:32.215] Dropping existing documents from system.replset [2021-10-22T01:22:32.379+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:22:32.379] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:22:32.383+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:22:32.383] Inserting oplogTruncateAfterPoint document [2021-10-22T01:22:32.383+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:22:32.383] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:22:32.384+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:22:32.383] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:22:32.384+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:22:32.384] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:22:32.396+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:22:32.396] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:22:32.396+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:22:32.396] Setting the minvalid document to [{_id ObjectID("61721258523370002becc52f")} {t -1} {ts {0 1}}] [2021-10-22T01:22:32.410+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:22:32.410] Inserted [{_id ObjectID("61721258523370002becc52f")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:22:32.410+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:22:32.410] Inserting new replica set configuration document. [2021-10-22T01:22:32.410+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:22:32.410] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:22:32.421+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:22:32.421] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:22:32.425+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:22:32.425] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:22:32.425+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:22:32.425] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:22:32.425+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:22:32.425] Initiating stopEphemeralMongo [2021-10-22T01:22:32.425+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:22:32.425] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:58498 (local=false) [2021-10-22T01:22:32.425+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:22:32.425] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:58498 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:22:32.762+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:22:32.762] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.732875ms. [2021-10-22T01:22:32.762+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:22:32.762] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:58498 (local=false) [2021-10-22T01:22:32.762+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:22:32.762] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 30.279µs. [2021-10-22T01:22:32.762+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:22:32.762] Successfully finished running stopEphemeralMongo [2021-10-22T01:22:32.762+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:22:32.762] Successfully stopped existing ephemeral mongod. [2021-10-22T01:22:32.762+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:22:32.762] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:22:32.762+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:22:32.762] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:22:32.799+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:22:32.799] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:22:32.799+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:22:32.799] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39019 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:22:33.011+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:33.011] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:33.165+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:22:33.165] clusterConfig unchanged [2021-10-22T01:22:33.991+0000] [.error] [src/action/start.go:func1:93] [103] [01:22:33.990] Error starting mongod : [01:22:33.990] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39019,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39019 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 30099 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:22:32.862+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:22:32.862+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:22:32.862+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39019,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:22:32.863+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:22:32.866+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:22:32.869+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:22:32.900+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:22:32.900+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:22:32.911+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:22:32.961+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:22:32.961+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:22:33.490+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865753:490557][30099:0x7f6e92b60bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:22:33.596+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865753:596616][30099:0x7f6e92b60bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:22:33.696+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865753:696158][30099:0x7f6e92b60bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T01:22:33.806+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865753:806733][30099:0x7f6e92b60bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:22:33.877+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865753:877840][30099:0x7f6e92b60bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:22:33.941+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865753:941297][30099:0x7f6e92b60bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:22:33.941+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865753:941369][30099:0x7f6e92b60bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:22:33.943+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:22:33.991+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:22:33.991] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39019 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:22:33.990] Error starting mongod : [01:22:33.990] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39019,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39019 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 30099 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:22:32.862+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:22:32.862+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:22:32.862+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39019,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:22:32.863+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:22:32.866+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:22:32.869+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:22:32.900+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:22:32.900+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:22:32.911+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:22:32.961+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:22:32.961+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:22:33.490+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865753:490557][30099:0x7f6e92b60bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:22:33.596+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865753:596616][30099:0x7f6e92b60bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:22:33.991+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:22:33.991] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:22:33.991] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39019 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:22:33.990] Error starting mongod : [01:22:33.990] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39019,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39019 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 30099 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:22:32.862+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:22:32.862+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:22:32.862+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39019,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:22:32.863+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:22:32.866+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:22:32.869+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:22:32.900+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:22:32.900+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:22:32.911+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:22:32.961+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:22:32.961+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:22:33.490+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865753:490557][30099:0x7f6e92b60bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:22:33.596+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:22:33.991+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:22:33.991] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:22:33.991+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:22:33.991] Initiating stopEphemeralMongo [2021-10-22T01:22:33.991+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:22:33.991] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:22:33.992+0000] [.error] [src/director/director.go:executePlan:962] [01:22:33.992] Failed to apply action. Result = : [01:22:33.991] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:22:33.991] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39019 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:22:33.990] Error starting mongod : [01:22:33.990] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39019,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39019 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 30099 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:22:32.862+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:22:32.862+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:22:32.862+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39019,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:22:32.863+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:22:32.866+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:22:32.869+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:22:32.900+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:22:32.900+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:22:32.911+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:22:32.961+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:22:32.961+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:22:33.490+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865753:490557][30099:0x7f6e92b60bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:22:33.992+0000] [.error] [src/director/director.go:planAndExecute:567] [01:22:33.992] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:22:33.992] Failed to apply action. Result = : [01:22:33.991] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:22:33.991] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39019 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:22:33.990] Error starting mongod : [01:22:33.990] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39019,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39019 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 30099 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:22:32.862+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:22:32.862+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:22:32.862+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39019,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:22:32.863+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:22:32.866+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:22:32.869+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:22:32.900+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:22:32.900+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:22:32.911+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:22:32.961+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:22:32.961+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:22:33.490+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:22:33.992+0000] [.error] [src/director/director.go:mainLoop:399] [01:22:33.992] Failed to planAndExecute : [01:22:33.992] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:22:33.992] Failed to apply action. Result = : [01:22:33.991] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:22:33.991] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39019 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:22:33.990] Error starting mongod : [01:22:33.990] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39019,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39019 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 30099 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:22:32.862+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:22:32.862+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:22:32.862+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39019,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:22:32.863+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:22:32.866+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:22:32.869+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:22:32.900+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:22:32.900+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:22:32.911+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:22:32.961+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:22:32.961+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:22:33.490+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:22:34.011+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:34.011] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:34.419+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:22:34.418] clusterConfig unchanged [2021-10-22T01:22:35.011+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:35.011] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:36.012+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [01:22:36.012] lsblk cache invalidated, running lsblk command [2021-10-22T01:22:36.014+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:36.014] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:37.012+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:37.012] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:38.019+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:38.019] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:39.019+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:39.019] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:40.017+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:40.017] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:41.019+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:41.019] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:41.321+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:22:41.321] memory:rss=52797440 alloc=12600344 totalalloc=5324391544 mallocs=68043474 frees=67944895 heapinuse=16678912 heapobjects=98579 gcpausetotalns=1493594228 gcpauselastns=85293 gcnum=837 utimens=99774425000 stimens=34572178000 minflt=269182 majflt=405 [2021-10-22T01:22:42.020+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:42.020] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:42.944+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:22:42.944] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.559604649s [2021-10-22T01:22:43.024+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:43.024] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:44.025+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:44.025] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:44.541+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:22:44.541] clusterConfig unchanged [2021-10-22T01:22:45.021+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:45.021] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:46.020+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:46.020] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:47.020+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:47.020] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:48.021+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:48.021] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:49.021+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:49.021] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:50.022+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:50.022] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:51.022+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:51.022] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:52.023+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:52.023] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:53.024+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:53.024] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:54.042+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:54.042] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:54.654+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:22:54.654] clusterConfig unchanged [2021-10-22T01:22:55.025+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:55.025] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:56.026+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:56.026] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:57.026+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:57.026] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:58.026+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:58.026] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:22:59.028+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:22:59.028] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:00.033+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:00.033] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:01.028+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:01.028] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:02.029+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:02.029] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:03.030+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:03.030] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:04.030+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:04.030] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:04.774+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:23:04.774] clusterConfig unchanged [2021-10-22T01:23:05.034+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:05.034] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:06.032+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:06.032] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:07.031+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:07.031] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:08.032+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:08.032] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:09.033+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:09.033] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:10.034+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:10.034] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:11.034+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:11.034] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:12.034+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:12.034] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:13.035+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:13.035] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:14.036+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:14.036] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:14.867+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:23:14.867] clusterConfig unchanged [2021-10-22T01:23:15.037+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:15.037] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:16.042+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:16.042] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:17.043+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:17.043] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:18.038+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:18.038] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:19.038+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:19.038] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:20.039+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:20.039] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:21.039+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:21.039] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:22.040+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:22.040] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:23.041+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:23.041] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:24.043+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:24.043] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:24.969+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:23:24.969] clusterConfig unchanged [2021-10-22T01:23:25.045+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:25.045] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:26.044+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:26.044] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:27.044+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:27.044] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:28.051+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:28.051] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:29.046+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:29.046] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:30.047+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:30.047] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:31.047+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:31.047] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:32.048+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:32.048] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:33.053+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:33.053] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:34.062+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:34.062] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:34.171+0000] [.info] [src/director/director.go:computePlan:280] [01:23:34.171] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:23:34.171+0000] [.info] [src/director/director.go:tracef:794] [01:23:34.171] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:23:34.171+0000] [.info] [src/director/director.go:tracef:794] [01:23:34.171] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:23:34.171+0000] [.info] [src/director/director.go:tracef:794] [01:23:34.171] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:23:34.171+0000] [.info] [src/director/director.go:tracef:794] [01:23:34.171] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:23:34.172+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:23:34.172] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:23:34.328+0000] [.info] [src/director/director.go:tracef:794] [01:23:34.328] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:23:34.328+0000] [.info] [src/director/director.go:tracef:794] [01:23:34.328] because [2021-10-22T01:23:34.328+0000] [.info] [src/director/director.go:tracef:794] [01:23:34.328] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:23:34.328+0000] [.info] [src/director/director.go:tracef:794] [01:23:34.328] because ['currentState.CanBounceStop' = true] [2021-10-22T01:23:34.482+0000] [.info] [src/director/director.go:tracef:794] [01:23:34.482] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:23:34.482+0000] [.info] [src/director/director.go:tracef:794] [01:23:34.482] because [2021-10-22T01:23:34.482+0000] [.info] [src/director/director.go:tracef:794] [01:23:34.482] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:23:34.482+0000] [.info] [src/director/director.go:tracef:794] [01:23:34.482] because ['currentState.Up' = false] [2021-10-22T01:23:34.483+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:23:34.483] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:23:34.483+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:23:34.483] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:23:34.569+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:23:34.569] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:23:34.578+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:23:34.578] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:23:34.578+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:23:34.578] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:23:34.579+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:23:34.579] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:23:34.580+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:23:34.580] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:23:34.580+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:23:34.580] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:23:34.580+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:23:34.580] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:23:34.581+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:23:34.581] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:23:34.771+0000] [.info] [src/director/director.go:tracef:794] [01:23:34.771] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:23:34.771+0000] [.info] [src/director/director.go:tracef:794] [01:23:34.771] because ['currentState.Up' = false] [2021-10-22T01:23:34.976+0000] [.info] [src/director/director.go:tracef:794] [01:23:34.976] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:23:34.976+0000] [.info] [src/director/director.go:tracef:794] [01:23:34.976] because ['currentState.Up' = false] [2021-10-22T01:23:34.976+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:23:34.976] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:23:35.084+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:35.084] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:35.169+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:23:35.169] clusterConfig unchanged [2021-10-22T01:23:35.227+0000] [.info] [src/director/director.go:tracef:794] [01:23:35.227] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:23:35.227+0000] [.info] [src/director/director.go:tracef:794] [01:23:35.227] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:23:35.462+0000] [.info] [src/director/director.go:tracef:794] [01:23:35.462] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:23:35.462+0000] [.info] [src/director/director.go:tracef:794] [01:23:35.462] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:23:35.462+0000] [.info] [src/director/director.go:tracef:794] [01:23:35.462] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:23:35.462+0000] [.info] [src/director/director.go:tracef:794] [01:23:35.462] because ['currentState.Up' = false] [2021-10-22T01:23:35.462+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:23:35.462] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:23:35.462+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:23:35.462] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:23:35.499+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:23:35.499] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:23:35.500+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:23:35.499] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52827 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:23:36.062+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:36.062] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:36.213+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:23:36.213] ...Finished start [pid=49460] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52827,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:23:36.307+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:23:36.307] clusterConfig unchanged [2021-10-22T01:23:36.334+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:23:36.334] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 120.361386ms. [2021-10-22T01:23:36.334+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:23:36.334] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52827 (local=false) [2021-10-22T01:23:36.334+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:23:36.334] Dropping existing documents from system.replset [2021-10-22T01:23:36.466+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:23:36.466] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:23:36.473+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:23:36.473] Inserting oplogTruncateAfterPoint document [2021-10-22T01:23:36.473+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:23:36.473] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:23:36.474+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:23:36.474] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:23:36.474+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:23:36.474] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:23:36.488+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:23:36.488] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:23:36.488+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:23:36.488] Setting the minvalid document to [{_id ObjectID("61721298523370002becc53d")} {t -1} {ts {0 1}}] [2021-10-22T01:23:36.501+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:23:36.501] Inserted [{_id ObjectID("61721298523370002becc53d")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:23:36.501+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:23:36.501] Inserting new replica set configuration document. [2021-10-22T01:23:36.501+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:23:36.501] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:23:36.514+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:23:36.514] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:23:36.518+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:23:36.518] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:23:36.518+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:23:36.518] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:23:36.518+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:23:36.518] Initiating stopEphemeralMongo [2021-10-22T01:23:36.518+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:23:36.518] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52827 (local=false) [2021-10-22T01:23:36.518+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:23:36.518] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52827 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:23:36.914+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:23:36.913] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.93734ms. [2021-10-22T01:23:36.914+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:23:36.914] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52827 (local=false) [2021-10-22T01:23:36.914+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:23:36.914] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 40.223µs. [2021-10-22T01:23:36.914+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:23:36.914] Successfully finished running stopEphemeralMongo [2021-10-22T01:23:36.914+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:23:36.914] Successfully stopped existing ephemeral mongod. [2021-10-22T01:23:36.914+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:23:36.914] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:23:36.914+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:23:36.914] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:23:36.951+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:23:36.951] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:23:36.952+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:23:36.952] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50515 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:23:37.064+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:37.064] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:37.426+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:23:37.426] clusterConfig unchanged [2021-10-22T01:23:38.069+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:38.068] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:38.222+0000] [.error] [src/action/start.go:func1:93] [103] [01:23:38.222] Error starting mongod : [01:23:38.222] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50515,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50515 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:23:36.981Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 50973 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:23:37.005+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:23:37.005+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:23:37.005+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50515,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:23:37.007+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:23:37.008+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:23:37.008+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:23:37.029+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:23:37.029+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:23:37.030+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:23:37.080+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:23:37.080+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:23:37.653+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865817:653904][50973:0x7f82a2b51bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:23:37.784+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865817:784666][50973:0x7f82a2b51bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:23:37.888+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865817:888705][50973:0x7f82a2b51bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:23:38.027+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865818:27902][50973:0x7f82a2b51bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:23:38.106+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865818:106394][50973:0x7f82a2b51bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:23:38.171+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865818:171700][50973:0x7f82a2b51bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:23:38.171+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865818:171780][50973:0x7f82a2b51bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:23:38.222+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:23:38.222] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:50515 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:23:38.222] Error starting mongod : [01:23:38.222] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50515,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50515 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:23:36.981Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 50973 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:23:37.005+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:23:37.005+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:23:37.005+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50515,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:23:37.007+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:23:37.008+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:23:37.008+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:23:37.029+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:23:37.029+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:23:37.030+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:23:37.080+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:23:37.080+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:23:37.653+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865817:653904][50973:0x7f82a2b51bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:23:37.784+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:23:38.223+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:23:38.223] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:23:38.222] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:50515 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:23:38.222] Error starting mongod : [01:23:38.222] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50515,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50515 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:23:36.981Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 50973 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:23:37.005+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:23:37.005+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:23:37.005+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50515,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:23:37.007+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:23:37.008+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:23:37.008+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:23:37.029+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:23:37.029+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:23:37.030+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:23:37.080+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:23:37.080+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:23:37.653+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865817:653904][50973:0x7f82a2b51bc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:23:38.223+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:23:38.223] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:23:38.223+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:23:38.223] Initiating stopEphemeralMongo [2021-10-22T01:23:38.223+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:23:38.223] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:23:38.223+0000] [.error] [src/director/director.go:executePlan:962] [01:23:38.223] Failed to apply action. Result = : [01:23:38.223] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:23:38.222] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:50515 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:23:38.222] Error starting mongod : [01:23:38.222] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50515,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50515 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:23:36.981Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 50973 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:23:37.005+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:23:37.005+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:23:37.005+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50515,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:23:37.007+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:23:37.008+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:23:37.008+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:23:37.029+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:23:37.029+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:23:37.030+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:23:37.080+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:23:37.080+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:23:37.653+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:23:38.224+0000] [.error] [src/director/director.go:planAndExecute:567] [01:23:38.224] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:23:38.223] Failed to apply action. Result = : [01:23:38.223] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:23:38.222] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:50515 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:23:38.222] Error starting mongod : [01:23:38.222] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50515,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50515 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:23:36.981Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 50973 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:23:37.005+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:23:37.005+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:23:37.005+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50515,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:23:37.007+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:23:37.008+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:23:37.008+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:23:37.029+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:23:37.029+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:23:37.030+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:23:37.080+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:23:37.080+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:23:38.224+0000] [.error] [src/director/director.go:mainLoop:399] [01:23:38.224] Failed to planAndExecute : [01:23:38.224] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:23:38.223] Failed to apply action. Result = : [01:23:38.223] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:23:38.222] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:50515 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:23:38.222] Error starting mongod : [01:23:38.222] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50515,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50515 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:23:36.981Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 50973 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:23:37.005+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:23:37.005+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:23:37.005+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50515,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:23:37.007+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:23:37.008+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:23:37.008+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:23:37.029+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:23:37.029+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:23:37.030+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:23:37.080+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:23:37.080+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:23:38.675+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:23:38.675] clusterConfig unchanged [2021-10-22T01:23:39.065+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:39.065] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:40.082+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:40.082] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:41.067+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:41.067] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:41.322+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:23:41.322] memory:rss=54816768 alloc=9000632 totalalloc=5410269512 mallocs=69141975 frees=69077608 heapinuse=15777792 heapobjects=64367 gcpausetotalns=1497280049 gcpauselastns=116939 gcnum=850 utimens=101635644000 stimens=35299901000 minflt=273641 majflt=405 [2021-10-22T01:23:42.066+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:42.066] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:43.068+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:43.068] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:44.069+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:44.069] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:44.252+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [01:23:44.252] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50515 (local=false) because it is not up [2021-10-22T01:23:45.069+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:45.069] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:46.072+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:46.072] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:47.070+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:47.070] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:47.239+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:23:47.239] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.658139867s [2021-10-22T01:23:48.073+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:48.073] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:48.806+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:23:48.806] clusterConfig unchanged [2021-10-22T01:23:49.069+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:49.069] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:50.068+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:50.068] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:51.069+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:51.069] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:52.069+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:52.069] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:53.070+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:53.070] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:54.071+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:54.071] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:55.071+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:55.071] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:56.072+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:56.072] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:57.072+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:57.072] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:58.088+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:58.088] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:23:58.938+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:23:58.938] clusterConfig unchanged [2021-10-22T01:23:59.080+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:23:59.080] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:00.074+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:00.074] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:01.074+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:01.074] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:02.074+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:02.074] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:03.075+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:03.075] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:04.075+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:04.075] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:05.075+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:05.075] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:06.076+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:06.076] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:07.077+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:07.077] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:08.077+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:08.077] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:09.050+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:24:09.050] clusterConfig unchanged [2021-10-22T01:24:09.077+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:09.077] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:10.078+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:10.078] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:11.079+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:11.079] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:12.080+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:12.080] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:13.080+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:13.080] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:14.080+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:14.080] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:15.081+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:15.081] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:16.081+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:16.081] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:17.082+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:17.082] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:18.083+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:18.083] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:19.082+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:19.082] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:19.182+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:24:19.182] clusterConfig unchanged [2021-10-22T01:24:20.083+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:20.083] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:21.089+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:21.089] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:22.083+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:22.083] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:23.086+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:23.086] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:24.087+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:24.087] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:25.085+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:25.085] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:26.089+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:26.089] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:27.092+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:27.092] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:28.091+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:28.091] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:29.092+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:29.092] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:29.294+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:24:29.294] clusterConfig unchanged [2021-10-22T01:24:30.092+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:30.092] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:31.093+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:31.093] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:32.094+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:32.094] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:33.100+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:33.100] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:34.095+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:34.095] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:35.095+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:35.095] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:36.096+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:36.096] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:37.097+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:37.097] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:38.097+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:38.097] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:38.391+0000] [.info] [src/director/director.go:computePlan:280] [01:24:38.391] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:24:38.391+0000] [.info] [src/director/director.go:tracef:794] [01:24:38.391] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:24:38.391+0000] [.info] [src/director/director.go:tracef:794] [01:24:38.391] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:24:38.391+0000] [.info] [src/director/director.go:tracef:794] [01:24:38.391] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:24:38.391+0000] [.info] [src/director/director.go:tracef:794] [01:24:38.391] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:24:38.391+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:24:38.391] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:24:38.564+0000] [.info] [src/director/director.go:tracef:794] [01:24:38.564] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:24:38.564+0000] [.info] [src/director/director.go:tracef:794] [01:24:38.564] because [2021-10-22T01:24:38.564+0000] [.info] [src/director/director.go:tracef:794] [01:24:38.564] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:24:38.564+0000] [.info] [src/director/director.go:tracef:794] [01:24:38.564] because ['currentState.CanBounceStop' = true] [2021-10-22T01:24:38.710+0000] [.info] [src/director/director.go:tracef:794] [01:24:38.710] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:24:38.710+0000] [.info] [src/director/director.go:tracef:794] [01:24:38.710] because [2021-10-22T01:24:38.710+0000] [.info] [src/director/director.go:tracef:794] [01:24:38.710] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:24:38.710+0000] [.info] [src/director/director.go:tracef:794] [01:24:38.710] because ['currentState.Up' = false] [2021-10-22T01:24:38.710+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:24:38.710] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:24:38.710+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:24:38.710] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:24:38.795+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:24:38.795] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:24:38.796+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:24:38.796] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:24:38.803+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:24:38.803] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:24:38.807+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:24:38.807] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:24:38.807+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:24:38.807] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:24:38.808+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:24:38.808] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:24:38.812+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:24:38.811] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:24:39.025+0000] [.info] [src/director/director.go:tracef:794] [01:24:39.025] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:24:39.025+0000] [.info] [src/director/director.go:tracef:794] [01:24:39.025] because ['currentState.Up' = false] [2021-10-22T01:24:39.114+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:39.114] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:39.243+0000] [.info] [src/director/director.go:tracef:794] [01:24:39.243] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:24:39.243+0000] [.info] [src/director/director.go:tracef:794] [01:24:39.243] because ['currentState.Up' = false] [2021-10-22T01:24:39.244+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:24:39.243] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:24:39.447+0000] [.info] [src/director/director.go:tracef:794] [01:24:39.447] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:24:39.447+0000] [.info] [src/director/director.go:tracef:794] [01:24:39.447] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:24:39.494+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:24:39.494] clusterConfig unchanged [2021-10-22T01:24:39.672+0000] [.info] [src/director/director.go:tracef:794] [01:24:39.672] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:24:39.672+0000] [.info] [src/director/director.go:tracef:794] [01:24:39.672] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:24:39.672+0000] [.info] [src/director/director.go:tracef:794] [01:24:39.672] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:24:39.672+0000] [.info] [src/director/director.go:tracef:794] [01:24:39.672] because ['currentState.Up' = false] [2021-10-22T01:24:39.672+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:24:39.672] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:24:39.672+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:24:39.672] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:24:39.702+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:24:39.702] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:24:39.703+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:24:39.703] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57279 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:24:40.103+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:40.103] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:40.483+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:24:40.483] ...Finished start [pid=70164] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57279,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:24:40.617+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:24:40.617] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 132.467179ms. [2021-10-22T01:24:40.617+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:24:40.617] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57279 (local=false) [2021-10-22T01:24:40.617+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:24:40.617] Dropping existing documents from system.replset [2021-10-22T01:24:40.619+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:24:40.619] clusterConfig unchanged [2021-10-22T01:24:40.767+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:24:40.767] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:24:40.771+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:24:40.771] Inserting oplogTruncateAfterPoint document [2021-10-22T01:24:40.771+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:24:40.771] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:24:40.772+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:24:40.772] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:24:40.772+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:24:40.772] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:24:40.784+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:24:40.784] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:24:40.784+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:24:40.784] Setting the minvalid document to [{_id ObjectID("617212d8523370002becc54b")} {t -1} {ts {0 1}}] [2021-10-22T01:24:40.797+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:24:40.797] Inserted [{_id ObjectID("617212d8523370002becc54b")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:24:40.797+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:24:40.797] Inserting new replica set configuration document. [2021-10-22T01:24:40.797+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:24:40.797] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:24:40.811+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:24:40.811] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:24:40.814+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:24:40.814] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:24:40.814+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:24:40.814] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:24:40.814+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:24:40.814] Initiating stopEphemeralMongo [2021-10-22T01:24:40.814+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:24:40.814] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57279 (local=false) [2021-10-22T01:24:40.814+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:24:40.814] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57279 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:24:41.101+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:41.101] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:41.134+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:24:41.134] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.079611ms. [2021-10-22T01:24:41.134+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:24:41.134] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57279 (local=false) [2021-10-22T01:24:41.134+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:24:41.134] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.923µs. [2021-10-22T01:24:41.134+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:24:41.134] Successfully finished running stopEphemeralMongo [2021-10-22T01:24:41.135+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:24:41.135] Successfully stopped existing ephemeral mongod. [2021-10-22T01:24:41.135+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:24:41.135] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:24:41.135+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:24:41.135] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:24:41.168+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:24:41.168] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:24:41.169+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:24:41.169] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53022 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:24:41.324+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:24:41.324] memory:rss=52838400 alloc=10806712 totalalloc=5494427048 mallocs=70239125 frees=70155940 heapinuse=15638528 heapobjects=83185 gcpausetotalns=1498658969 gcpauselastns=53566 gcnum=862 utimens=103338706000 stimens=35962480000 minflt=277607 majflt=405 [2021-10-22T01:24:41.750+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:24:41.750] clusterConfig unchanged [2021-10-22T01:24:42.104+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:42.104] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:42.362+0000] [.error] [src/action/start.go:func1:93] [103] [01:24:42.362] Error starting mongod : [01:24:42.362] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53022,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53022 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:24:41.198Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 71720 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:24:41.229+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:24:41.229+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:24:41.229+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53022,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:24:41.231+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:24:41.234+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:24:41.235+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:24:41.264+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:24:41.264+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:24:41.264+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:24:41.310+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:24:41.310+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:24:41.853+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865881:853299][71720:0x7fcd1138ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:24:41.958+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865881:958268][71720:0x7fcd1138ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:24:42.058+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865882:58470][71720:0x7fcd1138ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:24:42.169+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865882:169268][71720:0x7fcd1138ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:24:42.251+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865882:251569][71720:0x7fcd1138ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:24:42.311+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865882:311910][71720:0x7fcd1138ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:24:42.312+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865882:311996][71720:0x7fcd1138ebc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:24:42.362+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:24:42.362] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53022 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:24:42.362] Error starting mongod : [01:24:42.362] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53022,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53022 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:24:41.198Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 71720 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:24:41.229+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:24:41.229+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:24:41.229+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53022,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:24:41.231+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:24:41.234+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:24:41.235+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:24:41.264+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:24:41.264+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:24:41.264+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:24:41.310+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:24:41.310+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:24:41.853+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865881:853299][71720:0x7fcd1138ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:24:41.958+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:24:42.362+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:24:42.362] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:24:42.362] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53022 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:24:42.362] Error starting mongod : [01:24:42.362] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53022,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53022 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:24:41.198Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 71720 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:24:41.229+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:24:41.229+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:24:41.229+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53022,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:24:41.231+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:24:41.234+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:24:41.235+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:24:41.264+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:24:41.264+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:24:41.264+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:24:41.310+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:24:41.310+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:24:41.853+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865881:853299][71720:0x7fcd1138ebc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:24:42.363+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:24:42.363] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:24:42.363+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:24:42.363] Initiating stopEphemeralMongo [2021-10-22T01:24:42.363+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:24:42.363] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:24:42.364+0000] [.error] [src/director/director.go:executePlan:962] [01:24:42.364] Failed to apply action. Result = : [01:24:42.362] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:24:42.362] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53022 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:24:42.362] Error starting mongod : [01:24:42.362] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53022,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53022 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:24:41.198Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 71720 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:24:41.229+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:24:41.229+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:24:41.229+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53022,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:24:41.231+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:24:41.234+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:24:41.235+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:24:41.264+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:24:41.264+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:24:41.264+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:24:41.310+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:24:41.310+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:24:41.853+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:24:42.364+0000] [.error] [src/director/director.go:planAndExecute:567] [01:24:42.364] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:24:42.364] Failed to apply action. Result = : [01:24:42.362] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:24:42.362] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53022 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:24:42.362] Error starting mongod : [01:24:42.362] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53022,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53022 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:24:41.198Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 71720 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:24:41.229+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:24:41.229+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:24:41.229+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53022,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:24:41.231+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:24:41.234+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:24:41.235+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:24:41.264+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:24:41.264+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:24:41.264+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:24:41.310+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:24:41.310+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:24:42.364+0000] [.error] [src/director/director.go:mainLoop:399] [01:24:42.364] Failed to planAndExecute : [01:24:42.364] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:24:42.364] Failed to apply action. Result = : [01:24:42.362] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:24:42.362] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53022 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:24:42.362] Error starting mongod : [01:24:42.362] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53022,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53022 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:24:41.198Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 71720 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:24:41.229+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:24:41.229+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:24:41.229+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53022,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:24:41.231+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:24:41.234+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:24:41.235+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:24:41.264+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:24:41.264+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:24:41.264+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:24:41.310+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:24:41.310+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:24:42.981+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:24:42.981] clusterConfig unchanged [2021-10-22T01:24:43.118+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:43.118] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:44.103+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:44.102] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:45.104+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:45.104] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:46.102+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:46.102] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:47.104+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:47.104] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:48.106+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:48.106] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:49.106+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:49.106] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:50.107+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:50.107] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:51.106+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:51.106] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:51.121+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:24:51.121] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.309259046s [2021-10-22T01:24:52.106+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:52.106] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:53.107+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:53.107] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:53.111+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:24:53.111] clusterConfig unchanged [2021-10-22T01:24:54.107+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:54.107] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:55.108+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:55.108] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:56.109+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:56.109] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:57.109+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:57.109] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:58.110+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:58.110] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:24:59.110+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:24:59.110] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:00.110+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:00.110] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:01.117+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:01.117] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:02.113+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:02.113] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:03.113+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:03.113] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:03.329+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:25:03.329] clusterConfig unchanged [2021-10-22T01:25:04.114+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:04.114] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:05.115+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:05.115] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:06.115+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:06.115] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:07.116+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:07.116] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:08.116+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:08.116] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:09.117+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:09.117] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:10.117+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:10.117] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:11.118+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:11.118] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:12.122+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:12.122] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:13.119+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:13.119] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:13.470+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:25:13.470] clusterConfig unchanged [2021-10-22T01:25:14.120+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:14.120] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:15.122+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:15.122] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:16.120+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:16.120] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:17.121+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:17.121] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:18.122+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:18.122] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:19.123+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:19.123] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:20.123+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:20.123] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:21.123+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:21.123] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:22.124+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:22.124] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:23.124+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:23.124] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:23.572+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:25:23.572] clusterConfig unchanged [2021-10-22T01:25:24.125+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:24.125] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:25.125+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:25.125] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:26.126+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:26.126] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:27.127+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:27.127] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:28.128+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:28.128] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:29.129+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:29.129] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:30.129+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:30.129] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:31.129+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:31.129] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:32.130+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:32.130] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:33.130+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:33.130] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:33.713+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:25:33.713] clusterConfig unchanged [2021-10-22T01:25:34.131+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:34.131] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:35.138+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:35.138] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:36.132+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:36.132] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:37.133+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:37.133] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:38.134+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:38.134] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:39.134+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:39.134] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:40.135+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:40.135] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:41.136+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:41.136] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:41.325+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:25:41.325] memory:rss=52736000 alloc=10840480 totalalloc=5534169632 mallocs=70689149 frees=70602407 heapinuse=15458304 heapobjects=86742 gcpausetotalns=1499639424 gcpauselastns=77069 gcnum=868 utimens=104460323000 stimens=36564433000 minflt=281229 majflt=405 [2021-10-22T01:25:42.137+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:42.137] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:42.534+0000] [.info] [src/director/director.go:computePlan:280] [01:25:42.534] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:25:42.534+0000] [.info] [src/director/director.go:tracef:794] [01:25:42.534] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:25:42.534+0000] [.info] [src/director/director.go:tracef:794] [01:25:42.534] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:25:42.534+0000] [.info] [src/director/director.go:tracef:794] [01:25:42.534] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:25:42.534+0000] [.info] [src/director/director.go:tracef:794] [01:25:42.534] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:25:42.534+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:25:42.534] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:25:42.692+0000] [.info] [src/director/director.go:tracef:794] [01:25:42.692] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:25:42.692+0000] [.info] [src/director/director.go:tracef:794] [01:25:42.692] because [2021-10-22T01:25:42.692+0000] [.info] [src/director/director.go:tracef:794] [01:25:42.692] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:25:42.692+0000] [.info] [src/director/director.go:tracef:794] [01:25:42.692] because ['currentState.CanBounceStop' = true] [2021-10-22T01:25:42.852+0000] [.info] [src/director/director.go:tracef:794] [01:25:42.852] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:25:42.852+0000] [.info] [src/director/director.go:tracef:794] [01:25:42.852] because [2021-10-22T01:25:42.852+0000] [.info] [src/director/director.go:tracef:794] [01:25:42.852] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:25:42.852+0000] [.info] [src/director/director.go:tracef:794] [01:25:42.852] because ['currentState.Up' = false] [2021-10-22T01:25:42.852+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:25:42.852] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:25:42.852+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:25:42.852] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:25:42.949+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:25:42.949] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:25:42.957+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:25:42.957] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:25:42.958+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:25:42.958] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:25:42.958+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:25:42.958] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:25:42.959+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:25:42.959] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:25:42.959+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:25:42.959] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:25:42.959+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:25:42.959] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:25:42.961+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:25:42.961] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:25:43.162+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:43.162] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:43.177+0000] [.info] [src/director/director.go:tracef:794] [01:25:43.177] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:25:43.178+0000] [.info] [src/director/director.go:tracef:794] [01:25:43.178] because ['currentState.Up' = false] [2021-10-22T01:25:43.385+0000] [.info] [src/director/director.go:tracef:794] [01:25:43.385] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:25:43.385+0000] [.info] [src/director/director.go:tracef:794] [01:25:43.385] because ['currentState.Up' = false] [2021-10-22T01:25:43.385+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:25:43.385] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:25:43.610+0000] [.info] [src/director/director.go:tracef:794] [01:25:43.610] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:25:43.610+0000] [.info] [src/director/director.go:tracef:794] [01:25:43.610] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:25:43.818+0000] [.info] [src/director/director.go:tracef:794] [01:25:43.818] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:25:43.818+0000] [.info] [src/director/director.go:tracef:794] [01:25:43.818] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:25:43.818+0000] [.info] [src/director/director.go:tracef:794] [01:25:43.818] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:25:43.818+0000] [.info] [src/director/director.go:tracef:794] [01:25:43.818] because ['currentState.Up' = false] [2021-10-22T01:25:43.818+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:25:43.818] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:25:43.818+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:25:43.818] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:25:43.851+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:25:43.851] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:25:43.851+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:25:43.851] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51360 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:25:43.990+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:25:43.990] clusterConfig unchanged [2021-10-22T01:25:44.142+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:44.142] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:44.561+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:25:44.561] ...Finished start [pid=16890] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51360,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:25:44.674+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:25:44.674] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 113.342369ms. [2021-10-22T01:25:44.674+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:25:44.674] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51360 (local=false) [2021-10-22T01:25:44.674+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:25:44.674] Dropping existing documents from system.replset [2021-10-22T01:25:44.794+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:25:44.794] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:25:44.798+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:25:44.798] Inserting oplogTruncateAfterPoint document [2021-10-22T01:25:44.798+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:25:44.798] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:25:44.799+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:25:44.799] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:25:44.799+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:25:44.799] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:25:44.814+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:25:44.814] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:25:44.814+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:25:44.814] Setting the minvalid document to [{_id ObjectID("61721318523370002becc559")} {t -1} {ts {0 1}}] [2021-10-22T01:25:44.827+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:25:44.827] Inserted [{_id ObjectID("61721318523370002becc559")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:25:44.827+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:25:44.827] Inserting new replica set configuration document. [2021-10-22T01:25:44.827+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:25:44.827] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:25:44.840+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:25:44.840] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:25:44.843+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:25:44.843] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:25:44.843+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:25:44.843] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:25:44.843+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:25:44.843] Initiating stopEphemeralMongo [2021-10-22T01:25:44.843+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:25:44.843] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51360 (local=false) [2021-10-22T01:25:44.843+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:25:44.843] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51360 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:25:45.125+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:25:45.125] clusterConfig unchanged [2021-10-22T01:25:45.141+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:45.141] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:45.219+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:25:45.219] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.186891ms. [2021-10-22T01:25:45.219+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:25:45.219] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51360 (local=false) [2021-10-22T01:25:45.219+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:25:45.219] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.375µs. [2021-10-22T01:25:45.219+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:25:45.219] Successfully finished running stopEphemeralMongo [2021-10-22T01:25:45.219+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:25:45.219] Successfully stopped existing ephemeral mongod. [2021-10-22T01:25:45.219+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:25:45.219] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:25:45.219+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:25:45.219] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:25:45.255+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:25:45.255] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:25:45.256+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:25:45.256] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37357 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:25:46.144+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:46.144] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:46.248+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:25:46.248] clusterConfig unchanged [2021-10-22T01:25:46.430+0000] [.error] [src/action/start.go:func1:93] [103] [01:25:46.430] Error starting mongod : [01:25:46.430] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37357,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37357 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 18420 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:25:45.310+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:25:45.310+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:25:45.310+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37357,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:25:45.312+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:25:45.314+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:25:45.314+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:25:45.357+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:25:45.357+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:25:45.358+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:25:45.435+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:25:45.435+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:25:45.923+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865945:923343][18420:0x7fde66dccbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:25:46.026+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865946:26004][18420:0x7fde66dccbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:25:46.126+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865946:126902][18420:0x7fde66dccbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:25:46.239+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865946:239152][18420:0x7fde66dccbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:25:46.310+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865946:310069][18420:0x7fde66dccbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:25:46.372+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865946:372192][18420:0x7fde66dccbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:25:46.372+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865946:372262][18420:0x7fde66dccbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:25:46.373+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:25:46.430+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:25:46.430] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37357 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:25:46.430] Error starting mongod : [01:25:46.430] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37357,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37357 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 18420 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:25:45.310+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:25:45.310+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:25:45.310+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37357,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:25:45.312+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:25:45.314+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:25:45.314+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:25:45.357+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:25:45.357+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:25:45.358+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:25:45.435+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:25:45.435+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:25:45.923+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865945:923343][18420:0x7fde66dccbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:25:46.026+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865946:26004][18420:0x7fde66dccbc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:25:46.431+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:25:46.431] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:25:46.430] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37357 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:25:46.430] Error starting mongod : [01:25:46.430] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37357,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37357 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 18420 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:25:45.310+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:25:45.310+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:25:45.310+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37357,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:25:45.312+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:25:45.314+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:25:45.314+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:25:45.357+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:25:45.357+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:25:45.358+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:25:45.435+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:25:45.435+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:25:45.923+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865945:923343][18420:0x7fde66dccbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:25:46.026+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ct... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:25:46.431+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:25:46.431] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:25:46.431+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:25:46.431] Initiating stopEphemeralMongo [2021-10-22T01:25:46.431+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:25:46.431] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:25:46.431+0000] [.error] [src/director/director.go:executePlan:962] [01:25:46.431] Failed to apply action. Result = : [01:25:46.431] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:25:46.430] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37357 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:25:46.430] Error starting mongod : [01:25:46.430] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37357,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37357 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 18420 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:25:45.310+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:25:45.310+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:25:45.310+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37357,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:25:45.312+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:25:45.314+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:25:45.314+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:25:45.357+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:25:45.357+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:25:45.358+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:25:45.435+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:25:45.435+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:25:45.923+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634865945:923343][18420:0x7fde66dccbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:25:46.431+0000] [.error] [src/director/director.go:planAndExecute:567] [01:25:46.431] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:25:46.431] Failed to apply action. Result = : [01:25:46.431] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:25:46.430] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37357 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:25:46.430] Error starting mongod : [01:25:46.430] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37357,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37357 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 18420 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:25:45.310+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:25:45.310+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:25:45.310+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37357,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:25:45.312+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:25:45.314+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:25:45.314+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:25:45.357+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:25:45.357+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:25:45.358+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:25:45.435+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:25:45.435+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:25:45.923+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:25:46.431+0000] [.error] [src/director/director.go:mainLoop:399] [01:25:46.431] Failed to planAndExecute : [01:25:46.431] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:25:46.431] Failed to apply action. Result = : [01:25:46.431] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:25:46.430] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37357 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:25:46.430] Error starting mongod : [01:25:46.430] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37357,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37357 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 18420 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:25:45.310+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:25:45.310+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:25:45.310+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37357,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:25:45.312+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:25:45.314+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:25:45.314+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:25:45.357+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:25:45.357+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:25:45.358+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:25:45.435+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:25:45.435+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:25:45.923+00:00"},"s":"I", "c":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:25:47.149+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:47.149] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:47.498+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:25:47.498] clusterConfig unchanged [2021-10-22T01:25:48.144+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:48.144] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:49.144+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:49.144] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:50.145+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:50.145] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:51.146+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:51.146] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:52.147+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:52.147] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:53.148+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:53.148] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:54.149+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:54.149] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:55.150+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:55.150] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:55.273+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:25:55.273] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.311934065s [2021-10-22T01:25:56.148+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:56.148] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:57.149+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:57.149] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:57.620+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:25:57.620] clusterConfig unchanged [2021-10-22T01:25:58.152+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:58.152] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:25:59.150+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:25:59.150] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:00.151+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:00.151] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:01.151+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:01.151] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:02.152+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:02.152] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:03.152+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:03.152] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:04.153+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:04.153] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:05.159+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:05.159] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:06.155+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:06.155] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:07.155+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:07.155] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:07.840+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:26:07.840] clusterConfig unchanged [2021-10-22T01:26:08.155+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:08.155] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:09.156+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:09.156] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:10.157+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:10.157] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:11.157+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:11.157] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:12.157+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:12.157] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:13.158+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:13.158] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:14.159+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:14.159] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:15.160+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:15.160] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:16.161+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:16.161] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:17.168+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:17.168] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:17.934+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:26:17.934] clusterConfig unchanged [2021-10-22T01:26:18.164+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:18.164] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:18.559+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:26:18.559] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:26:18.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:18.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:26:18.562+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [01:26:18.562] Could not get version : [01:26:18.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:26:18.562+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [01:26:18.562] Error getting version for log lines. Skipping : [01:26:18.562] Could not get version : [01:26:18.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:26:19.162+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:19.162] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:20.163+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:20.163] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:21.162+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:21.162] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:22.162+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:22.162] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:23.163+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:23.163] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:24.164+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:24.164] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:25.164+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:25.164] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:26.168+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:26.168] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:27.165+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:27.165] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:28.042+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:26:28.042] clusterConfig unchanged [2021-10-22T01:26:28.166+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:28.166] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:29.167+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:29.167] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:30.167+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:30.167] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:31.168+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:31.168] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:32.169+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:32.169] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:33.170+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:33.170] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:34.170+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:34.170] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:35.171+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:35.171] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:36.172+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:36.172] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:37.173+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:37.173] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:38.177+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:38.177] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:38.252+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:26:38.252] clusterConfig unchanged [2021-10-22T01:26:39.174+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:39.174] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:40.174+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:40.174] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:41.175+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:41.175] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:41.325+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:26:41.325] memory:rss=52584448 alloc=9062992 totalalloc=5622803048 mallocs=71804210 frees=71736394 heapinuse=14925824 heapobjects=67816 gcpausetotalns=1503773929 gcpauselastns=74355 gcnum=881 utimens=106291236000 stimens=37322335000 minflt=285778 majflt=405 [2021-10-22T01:26:42.176+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:42.175] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:43.176+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:43.176] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:44.176+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:44.176] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:44.261+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [01:26:44.261] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37357 (local=false) because it is not up [2021-10-22T01:26:45.182+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:45.182] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:46.180+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:46.180] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:46.614+0000] [.info] [src/director/director.go:computePlan:280] [01:26:46.614] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:26:46.614+0000] [.info] [src/director/director.go:tracef:794] [01:26:46.614] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:26:46.614+0000] [.info] [src/director/director.go:tracef:794] [01:26:46.614] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:26:46.614+0000] [.info] [src/director/director.go:tracef:794] [01:26:46.614] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:26:46.614+0000] [.info] [src/director/director.go:tracef:794] [01:26:46.614] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:26:46.614+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:26:46.614] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:26:46.763+0000] [.info] [src/director/director.go:tracef:794] [01:26:46.763] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:26:46.763+0000] [.info] [src/director/director.go:tracef:794] [01:26:46.763] because [2021-10-22T01:26:46.763+0000] [.info] [src/director/director.go:tracef:794] [01:26:46.763] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:26:46.763+0000] [.info] [src/director/director.go:tracef:794] [01:26:46.763] because ['currentState.CanBounceStop' = true] [2021-10-22T01:26:46.923+0000] [.info] [src/director/director.go:tracef:794] [01:26:46.923] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:26:46.923+0000] [.info] [src/director/director.go:tracef:794] [01:26:46.923] because [2021-10-22T01:26:46.923+0000] [.info] [src/director/director.go:tracef:794] [01:26:46.923] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:26:46.923+0000] [.info] [src/director/director.go:tracef:794] [01:26:46.923] because ['currentState.Up' = false] [2021-10-22T01:26:46.924+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:26:46.924] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:26:46.924+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:26:46.924] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:26:47.006+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:26:47.006] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:26:47.014+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:26:47.014] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:26:47.014+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:26:47.014] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:26:47.015+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:26:47.015] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:26:47.016+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:26:47.016] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:26:47.016+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:26:47.016] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:26:47.016+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:26:47.016] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:26:47.018+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:26:47.018] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:26:47.181+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:47.181] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:47.231+0000] [.info] [src/director/director.go:tracef:794] [01:26:47.231] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:26:47.232+0000] [.info] [src/director/director.go:tracef:794] [01:26:47.231] because ['currentState.Up' = false] [2021-10-22T01:26:47.445+0000] [.info] [src/director/director.go:tracef:794] [01:26:47.445] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:26:47.445+0000] [.info] [src/director/director.go:tracef:794] [01:26:47.445] because ['currentState.Up' = false] [2021-10-22T01:26:47.445+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:26:47.445] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:26:47.645+0000] [.info] [src/director/director.go:tracef:794] [01:26:47.645] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:26:47.645+0000] [.info] [src/director/director.go:tracef:794] [01:26:47.645] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:26:47.842+0000] [.info] [src/director/director.go:tracef:794] [01:26:47.842] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:26:47.842+0000] [.info] [src/director/director.go:tracef:794] [01:26:47.842] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:26:47.842+0000] [.info] [src/director/director.go:tracef:794] [01:26:47.842] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:26:47.842+0000] [.info] [src/director/director.go:tracef:794] [01:26:47.842] because ['currentState.Up' = false] [2021-10-22T01:26:47.842+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:26:47.842] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:26:47.842+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:26:47.842] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:26:47.877+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:26:47.877] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:26:47.877+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:26:47.877] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57054 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:26:48.184+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:48.184] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:48.453+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:26:48.453] clusterConfig unchanged [2021-10-22T01:26:48.591+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:26:48.591] ...Finished start [pid=37528] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57054,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:26:48.729+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:26:48.729] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 137.176748ms. [2021-10-22T01:26:48.729+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:26:48.729] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57054 (local=false) [2021-10-22T01:26:48.729+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:26:48.729] Dropping existing documents from system.replset [2021-10-22T01:26:48.867+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:26:48.867] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:26:48.871+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:26:48.871] Inserting oplogTruncateAfterPoint document [2021-10-22T01:26:48.871+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:26:48.871] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:26:48.872+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:26:48.872] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:26:48.872+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:26:48.872] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:26:48.886+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:26:48.886] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:26:48.886+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:26:48.886] Setting the minvalid document to [{_id ObjectID("61721358523370002becc567")} {t -1} {ts {0 1}}] [2021-10-22T01:26:48.898+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:26:48.898] Inserted [{_id ObjectID("61721358523370002becc567")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:26:48.898+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:26:48.898] Inserting new replica set configuration document. [2021-10-22T01:26:48.898+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:26:48.898] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:26:48.911+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:26:48.911] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:26:48.915+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:26:48.915] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:26:48.915+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:26:48.915] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:26:48.915+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:26:48.915] Initiating stopEphemeralMongo [2021-10-22T01:26:48.915+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:26:48.915] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57054 (local=false) [2021-10-22T01:26:48.916+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:26:48.916] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57054 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:26:49.184+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:49.184] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:49.281+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:26:49.281] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.959328ms. [2021-10-22T01:26:49.281+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:26:49.281] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57054 (local=false) [2021-10-22T01:26:49.281+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:26:49.281] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.432µs. [2021-10-22T01:26:49.281+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:26:49.281] Successfully finished running stopEphemeralMongo [2021-10-22T01:26:49.281+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:26:49.281] Successfully stopped existing ephemeral mongod. [2021-10-22T01:26:49.281+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:26:49.281] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:26:49.281+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:26:49.281] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:26:49.315+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:26:49.315] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:26:49.316+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:26:49.316] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33495 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:26:49.596+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:26:49.596] clusterConfig unchanged [2021-10-22T01:26:50.187+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:50.187] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:50.505+0000] [.error] [src/action/start.go:func1:93] [103] [01:26:50.505] Error starting mongod : [01:26:50.505] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33495 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:26:49.348Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 39069 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:26:49.368+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:26:49.368+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:26:49.368+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:26:49.370+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:26:49.371+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:26:49.372+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:26:49.435+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:26:49.435+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:26:49.436+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:26:49.484+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:26:49.484+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:26:49.983+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866009:982954][39069:0x7f1804f9ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:26:50.087+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866010:87489][39069:0x7f1804f9ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:26:50.193+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866010:193826][39069:0x7f1804f9ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:26:50.310+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866010:310918][39069:0x7f1804f9ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:26:50.384+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866010:384003][39069:0x7f1804f9ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:26:50.448+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866010:448885][39069:0x7f1804f9ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:26:50.448+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866010:448981][39069:0x7f1804f9ebc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:26:50.505+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:26:50.505] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33495 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:26:50.505] Error starting mongod : [01:26:50.505] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33495 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:26:49.348Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 39069 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:26:49.368+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:26:49.368+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:26:49.368+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:26:49.370+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:26:49.371+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:26:49.372+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:26:49.435+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:26:49.435+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:26:49.436+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:26:49.484+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:26:49.484+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:26:49.983+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866009:982954][39069:0x7f1804f9ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:26:50.087+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:26:50.505+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:26:50.505] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:26:50.505] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33495 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:26:50.505] Error starting mongod : [01:26:50.505] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33495 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:26:49.348Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 39069 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:26:49.368+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:26:49.368+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:26:49.368+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:26:49.370+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:26:49.371+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:26:49.372+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:26:49.435+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:26:49.435+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:26:49.436+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:26:49.484+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:26:49.484+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:26:49.983+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866009:982954][39069:0x7f1804f9ebc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:26:50.505+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:26:50.505] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:26:50.505+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:26:50.505] Initiating stopEphemeralMongo [2021-10-22T01:26:50.505+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:26:50.505] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:26:50.508+0000] [.error] [src/director/director.go:executePlan:962] [01:26:50.508] Failed to apply action. Result = : [01:26:50.505] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:26:50.505] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33495 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:26:50.505] Error starting mongod : [01:26:50.505] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33495 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:26:49.348Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 39069 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:26:49.368+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:26:49.368+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:26:49.368+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:26:49.370+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:26:49.371+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:26:49.372+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:26:49.435+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:26:49.435+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:26:49.436+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:26:49.484+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:26:49.484+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:26:49.983+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:26:50.509+0000] [.error] [src/director/director.go:planAndExecute:567] [01:26:50.509] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:26:50.508] Failed to apply action. Result = : [01:26:50.505] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:26:50.505] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33495 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:26:50.505] Error starting mongod : [01:26:50.505] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33495 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:26:49.348Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 39069 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:26:49.368+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:26:49.368+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:26:49.368+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:26:49.370+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:26:49.371+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:26:49.372+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:26:49.435+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:26:49.435+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:26:49.436+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:26:49.484+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:26:49.484+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:26:50.509+0000] [.error] [src/director/director.go:mainLoop:399] [01:26:50.509] Failed to planAndExecute : [01:26:50.509] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:26:50.508] Failed to apply action. Result = : [01:26:50.505] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:26:50.505] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33495 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:26:50.505] Error starting mongod : [01:26:50.505] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33495 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:26:49.348Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 39069 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:26:49.368+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:26:49.368+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:26:49.368+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33495,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:26:49.370+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:26:49.371+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:26:49.372+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:26:49.435+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:26:49.435+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:26:49.436+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:26:49.484+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:26:49.484+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:26:50.856+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:26:50.856] clusterConfig unchanged [2021-10-22T01:26:51.187+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:51.187] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:52.187+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:52.187] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:53.187+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:53.187] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:54.187+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:54.187] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:55.202+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:55.202] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:56.186+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:56.186] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:57.195+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:57.195] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:58.195+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:58.195] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:59.198+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:26:59.198] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:26:59.440+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:26:59.440] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.422679719s [2021-10-22T01:27:00.196+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:00.196] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:00.997+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:27:00.997] clusterConfig unchanged [2021-10-22T01:27:01.197+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:01.197] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:02.198+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:02.198] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:03.199+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:03.199] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:04.200+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:04.200] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:05.207+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:05.207] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:06.202+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:06.202] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:07.202+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:07.202] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:08.203+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:08.203] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:09.204+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:09.204] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:10.204+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:10.204] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:11.090+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:27:11.090] clusterConfig unchanged [2021-10-22T01:27:11.205+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:11.205] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:12.206+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:12.206] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:13.206+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:13.206] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:14.207+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:14.207] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:15.207+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:15.207] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:16.213+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:16.213] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:17.208+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:17.208] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:18.208+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:18.208] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:19.209+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:19.209] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:20.210+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:20.210] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:21.210+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:21.210] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:21.231+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:27:21.231] clusterConfig unchanged [2021-10-22T01:27:22.211+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:22.211] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:23.211+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:23.211] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:24.212+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:24.212] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:25.214+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:25.214] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:26.213+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:26.213] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:27.220+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:27.220] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:27.562+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:27:27.562] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:27:27.563+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:27:27.563] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:27:28.214+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:28.214] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:29.214+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:29.214] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:30.215+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:30.215] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:31.215+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:31.215] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:31.362+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:27:31.362] clusterConfig unchanged [2021-10-22T01:27:32.216+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:32.216] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:33.216+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:33.216] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:34.217+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:34.217] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:35.217+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:35.217] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:36.220+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [01:27:36.220] lsblk cache invalidated, running lsblk command [2021-10-22T01:27:36.221+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:36.221] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:37.219+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:37.219] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:38.225+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:38.225] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:39.221+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:39.221] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:40.221+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:40.221] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:41.223+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:41.223] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:41.326+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:27:41.326] memory:rss=52125696 alloc=8919480 totalalloc=5711634096 mallocs=72926041 frees=72860190 heapinuse=14794752 heapobjects=65851 gcpausetotalns=1505252293 gcpauselastns=64554 gcnum=894 utimens=108176641000 stimens=38055699000 minflt=291127 majflt=405 [2021-10-22T01:27:41.465+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:27:41.465] clusterConfig unchanged [2021-10-22T01:27:42.222+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:42.222] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:43.223+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:43.223] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:44.224+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:44.224] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:45.224+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:45.224] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:46.230+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:46.230] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:47.225+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:47.225] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:48.225+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:48.225] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:49.226+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:49.226] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:50.226+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:50.226] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:50.676+0000] [.info] [src/director/director.go:computePlan:280] [01:27:50.676] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:27:50.676+0000] [.info] [src/director/director.go:tracef:794] [01:27:50.676] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:27:50.676+0000] [.info] [src/director/director.go:tracef:794] [01:27:50.676] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:27:50.677+0000] [.info] [src/director/director.go:tracef:794] [01:27:50.677] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:27:50.677+0000] [.info] [src/director/director.go:tracef:794] [01:27:50.677] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:27:50.677+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:27:50.677] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:27:50.833+0000] [.info] [src/director/director.go:tracef:794] [01:27:50.833] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:27:50.833+0000] [.info] [src/director/director.go:tracef:794] [01:27:50.833] because [2021-10-22T01:27:50.833+0000] [.info] [src/director/director.go:tracef:794] [01:27:50.833] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:27:50.833+0000] [.info] [src/director/director.go:tracef:794] [01:27:50.833] because ['currentState.CanBounceStop' = true] [2021-10-22T01:27:50.979+0000] [.info] [src/director/director.go:tracef:794] [01:27:50.979] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:27:50.979+0000] [.info] [src/director/director.go:tracef:794] [01:27:50.979] because [2021-10-22T01:27:50.979+0000] [.info] [src/director/director.go:tracef:794] [01:27:50.979] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:27:50.979+0000] [.info] [src/director/director.go:tracef:794] [01:27:50.979] because ['currentState.Up' = false] [2021-10-22T01:27:50.979+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:27:50.979] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:27:50.979+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:27:50.979] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:27:51.065+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:27:51.065] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:27:51.073+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:27:51.073] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:27:51.074+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:27:51.074] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:27:51.074+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:27:51.074] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:27:51.075+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:27:51.075] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:27:51.075+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:27:51.075] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:27:51.075+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:27:51.075] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:27:51.077+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:27:51.077] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:27:51.230+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:51.230] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:51.312+0000] [.info] [src/director/director.go:tracef:794] [01:27:51.312] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:27:51.312+0000] [.info] [src/director/director.go:tracef:794] [01:27:51.312] because ['currentState.Up' = false] [2021-10-22T01:27:51.533+0000] [.info] [src/director/director.go:tracef:794] [01:27:51.533] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:27:51.533+0000] [.info] [src/director/director.go:tracef:794] [01:27:51.533] because ['currentState.Up' = false] [2021-10-22T01:27:51.533+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:27:51.533] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:27:51.665+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:27:51.665] clusterConfig unchanged [2021-10-22T01:27:51.753+0000] [.info] [src/director/director.go:tracef:794] [01:27:51.753] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:27:51.753+0000] [.info] [src/director/director.go:tracef:794] [01:27:51.753] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:27:51.959+0000] [.info] [src/director/director.go:tracef:794] [01:27:51.959] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:27:51.959+0000] [.info] [src/director/director.go:tracef:794] [01:27:51.959] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:27:51.960+0000] [.info] [src/director/director.go:tracef:794] [01:27:51.960] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:27:51.960+0000] [.info] [src/director/director.go:tracef:794] [01:27:51.960] because ['currentState.Up' = false] [2021-10-22T01:27:51.960+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:27:51.960] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:27:51.960+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:27:51.960] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:27:51.991+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:27:51.991] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:27:51.991+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:27:51.991] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40484 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:27:52.233+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:52.232] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:52.713+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:27:52.712] ...Finished start [pid=58411] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40484,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:27:52.798+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:27:52.798] clusterConfig unchanged [2021-10-22T01:27:52.826+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:27:52.826] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 113.312526ms. [2021-10-22T01:27:52.826+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:27:52.826] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40484 (local=false) [2021-10-22T01:27:52.826+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:27:52.826] Dropping existing documents from system.replset [2021-10-22T01:27:52.962+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:27:52.962] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:27:52.966+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:27:52.966] Inserting oplogTruncateAfterPoint document [2021-10-22T01:27:52.966+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:27:52.966] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:27:52.967+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:27:52.967] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:27:52.967+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:27:52.967] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:27:52.979+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:27:52.979] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:27:52.979+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:27:52.979] Setting the minvalid document to [{_id ObjectID("61721398523370002becc575")} {t -1} {ts {0 1}}] [2021-10-22T01:27:52.990+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:27:52.990] Inserted [{_id ObjectID("61721398523370002becc575")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:27:52.990+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:27:52.990] Inserting new replica set configuration document. [2021-10-22T01:27:52.990+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:27:52.990] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:27:53.007+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:27:53.007] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:27:53.013+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:27:53.013] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:27:53.013+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:27:53.013] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:27:53.013+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:27:53.013] Initiating stopEphemeralMongo [2021-10-22T01:27:53.014+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:27:53.013] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40484 (local=false) [2021-10-22T01:27:53.014+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:27:53.014] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40484 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:27:53.235+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:53.235] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:53.343+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:27:53.343] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.473318ms. [2021-10-22T01:27:53.343+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:27:53.343] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40484 (local=false) [2021-10-22T01:27:53.343+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:27:53.343] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.952µs. [2021-10-22T01:27:53.343+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:27:53.343] Successfully finished running stopEphemeralMongo [2021-10-22T01:27:53.343+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:27:53.343] Successfully stopped existing ephemeral mongod. [2021-10-22T01:27:53.343+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:27:53.343] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:27:53.343+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:27:53.343] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:27:53.376+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:27:53.375] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:27:53.376+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:27:53.376] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46866 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:27:53.910+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:27:53.910] clusterConfig unchanged [2021-10-22T01:27:54.237+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:54.237] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:54.533+0000] [.error] [src/action/start.go:func1:93] [103] [01:27:54.533] Error starting mongod : [01:27:54.533] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46866,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46866 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 59916 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:27:53.440+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:27:53.440+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:27:53.440+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46866,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:27:53.442+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:27:53.444+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:27:53.446+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:27:53.479+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:27:53.479+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:27:53.479+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:27:53.528+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:27:53.528+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:27:54.039+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866074:39088][59916:0x7f6019fd5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:27:54.141+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866074:141820][59916:0x7f6019fd5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:27:54.241+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866074:241205][59916:0x7f6019fd5bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:27:54.356+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866074:356341][59916:0x7f6019fd5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:27:54.426+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866074:426329][59916:0x7f6019fd5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:27:54.483+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866074:483182][59916:0x7f6019fd5bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:27:54.483+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866074:483258][59916:0x7f6019fd5bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:27:54.484+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:27:54.533+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:27:54.533] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46866 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:27:54.533] Error starting mongod : [01:27:54.533] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46866,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46866 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 59916 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:27:53.440+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:27:53.440+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:27:53.440+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46866,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:27:53.442+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:27:53.444+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:27:53.446+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:27:53.479+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:27:53.479+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:27:53.479+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:27:53.528+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:27:53.528+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:27:54.039+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866074:39088][59916:0x7f6019fd5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:27:54.141+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866074:141820][59916:0x7f6019fd5bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:27:54.534+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:27:54.534] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:27:54.533] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46866 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:27:54.533] Error starting mongod : [01:27:54.533] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46866,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46866 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 59916 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:27:53.440+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:27:53.440+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:27:53.440+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46866,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:27:53.442+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:27:53.444+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:27:53.446+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:27:53.479+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:27:53.479+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:27:53.479+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:27:53.528+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:27:53.528+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:27:54.039+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866074:39088][59916:0x7f6019fd5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:27:54.141+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:27:54.535+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:27:54.535] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:27:54.535+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:27:54.535] Initiating stopEphemeralMongo [2021-10-22T01:27:54.535+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:27:54.535] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:27:54.540+0000] [.error] [src/director/director.go:executePlan:962] [01:27:54.540] Failed to apply action. Result = : [01:27:54.534] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:27:54.533] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46866 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:27:54.533] Error starting mongod : [01:27:54.533] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46866,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46866 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 59916 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:27:53.440+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:27:53.440+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:27:53.440+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46866,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:27:53.442+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:27:53.444+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:27:53.446+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:27:53.479+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:27:53.479+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:27:53.479+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:27:53.528+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:27:53.528+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:27:54.039+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866074:39088][59916:0x7f6019fd5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:27:54.540+0000] [.error] [src/director/director.go:planAndExecute:567] [01:27:54.540] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:27:54.540] Failed to apply action. Result = : [01:27:54.534] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:27:54.533] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46866 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:27:54.533] Error starting mongod : [01:27:54.533] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46866,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46866 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 59916 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:27:53.440+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:27:53.440+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:27:53.440+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46866,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:27:53.442+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:27:53.444+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:27:53.446+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:27:53.479+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:27:53.479+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:27:53.479+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:27:53.528+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:27:53.528+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:27:54.039+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:27:54.541+0000] [.error] [src/director/director.go:mainLoop:399] [01:27:54.541] Failed to planAndExecute : [01:27:54.540] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:27:54.540] Failed to apply action. Result = : [01:27:54.534] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:27:54.533] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46866 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:27:54.533] Error starting mongod : [01:27:54.533] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46866,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46866 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 59916 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:27:53.440+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:27:53.440+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:27:53.440+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46866,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:27:53.442+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:27:53.444+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:27:53.446+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:27:53.479+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:27:53.479+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:27:53.479+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:27:53.528+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:27:53.528+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:27:54.039+00:00"},"s":"I", "c":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:27:55.131+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:27:55.131] clusterConfig unchanged [2021-10-22T01:27:55.236+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:55.236] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:56.237+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:56.237] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:57.238+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:57.238] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:58.238+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:58.238] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:27:59.240+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:27:59.240] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:00.241+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:00.241] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:01.248+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:01.248] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:02.243+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:02.243] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:03.243+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:03.243] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:03.466+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:28:03.466] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.389253375s [2021-10-22T01:28:04.240+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:04.240] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:05.241+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:05.241] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:05.243+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:28:05.243] clusterConfig unchanged [2021-10-22T01:28:06.241+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:06.241] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:07.242+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:07.242] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:08.243+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:08.242] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:09.243+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:09.243] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:10.243+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:10.243] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:11.244+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:11.244] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:12.252+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:12.252] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:13.245+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:13.245] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:14.245+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:14.245] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:15.246+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:15.246] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:15.434+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:28:15.434] clusterConfig unchanged [2021-10-22T01:28:16.246+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:16.246] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:17.246+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:17.246] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:18.247+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:18.247] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:19.247+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:19.247] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:20.248+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:20.248] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:21.249+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:21.249] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:22.250+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:22.250] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:23.249+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:23.249] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:24.250+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:24.250] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:25.250+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:25.250] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:25.558+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:28:25.558] clusterConfig unchanged [2021-10-22T01:28:26.251+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:26.251] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:27.252+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:27.252] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:28.252+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:28.252] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:29.252+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:29.252] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:30.253+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:30.253] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:31.254+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:31.254] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:32.255+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:32.255] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:33.255+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:33.255] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:34.255+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:34.255] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:35.261+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:35.261] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:35.697+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:28:35.697] clusterConfig unchanged [2021-10-22T01:28:36.256+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:36.256] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:37.257+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:37.257] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:38.258+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:38.258] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:39.258+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:39.258] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:40.258+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:40.258] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:41.259+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:41.259] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:41.327+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:28:41.327] memory:rss=53309440 alloc=11067184 totalalloc=5800006904 mallocs=74049172 frees=73960729 heapinuse=15630336 heapobjects=88443 gcpausetotalns=1506960549 gcpauselastns=63733 gcnum=906 utimens=110059758000 stimens=38854350000 minflt=295958 majflt=405 [2021-10-22T01:28:42.259+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:42.259] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:43.261+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:43.261] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:44.260+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:44.260] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:45.261+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:45.261] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:45.830+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:28:45.830] clusterConfig unchanged [2021-10-22T01:28:46.261+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:46.261] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:47.261+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:47.261] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:48.262+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:48.262] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:49.263+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:49.263] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:50.263+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:50.263] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:51.264+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:51.264] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:52.264+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:52.264] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:53.265+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:53.265] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:54.265+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:54.265] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:54.705+0000] [.info] [src/director/director.go:computePlan:280] [01:28:54.705] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:28:54.705+0000] [.info] [src/director/director.go:tracef:794] [01:28:54.705] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:28:54.705+0000] [.info] [src/director/director.go:tracef:794] [01:28:54.705] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:28:54.706+0000] [.info] [src/director/director.go:tracef:794] [01:28:54.706] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:28:54.706+0000] [.info] [src/director/director.go:tracef:794] [01:28:54.706] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:28:54.706+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:28:54.706] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:28:54.861+0000] [.info] [src/director/director.go:tracef:794] [01:28:54.861] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:28:54.861+0000] [.info] [src/director/director.go:tracef:794] [01:28:54.861] because [2021-10-22T01:28:54.861+0000] [.info] [src/director/director.go:tracef:794] [01:28:54.861] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:28:54.861+0000] [.info] [src/director/director.go:tracef:794] [01:28:54.861] because ['currentState.CanBounceStop' = true] [2021-10-22T01:28:55.015+0000] [.info] [src/director/director.go:tracef:794] [01:28:55.015] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:28:55.015+0000] [.info] [src/director/director.go:tracef:794] [01:28:55.015] because [2021-10-22T01:28:55.016+0000] [.info] [src/director/director.go:tracef:794] [01:28:55.016] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:28:55.016+0000] [.info] [src/director/director.go:tracef:794] [01:28:55.016] because ['currentState.Up' = false] [2021-10-22T01:28:55.016+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:28:55.016] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:28:55.016+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:28:55.016] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:28:55.094+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:28:55.094] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:28:55.102+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:28:55.102] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:28:55.103+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:28:55.103] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:28:55.104+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:28:55.104] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:28:55.105+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:28:55.105] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:28:55.105+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:28:55.105] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:28:55.105+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:28:55.105] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:28:55.106+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:28:55.106] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:28:55.269+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:55.269] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:55.322+0000] [.info] [src/director/director.go:tracef:794] [01:28:55.322] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:28:55.322+0000] [.info] [src/director/director.go:tracef:794] [01:28:55.322] because ['currentState.Up' = false] [2021-10-22T01:28:55.519+0000] [.info] [src/director/director.go:tracef:794] [01:28:55.519] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:28:55.519+0000] [.info] [src/director/director.go:tracef:794] [01:28:55.519] because ['currentState.Up' = false] [2021-10-22T01:28:55.519+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:28:55.519] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:28:55.724+0000] [.info] [src/director/director.go:tracef:794] [01:28:55.724] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:28:55.724+0000] [.info] [src/director/director.go:tracef:794] [01:28:55.724] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:28:55.946+0000] [.info] [src/director/director.go:tracef:794] [01:28:55.946] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:28:55.946+0000] [.info] [src/director/director.go:tracef:794] [01:28:55.946] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:28:55.946+0000] [.info] [src/director/director.go:tracef:794] [01:28:55.946] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:28:55.946+0000] [.info] [src/director/director.go:tracef:794] [01:28:55.946] because ['currentState.Up' = false] [2021-10-22T01:28:55.946+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:28:55.946] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:28:55.946+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:28:55.946] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:28:55.982+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:28:55.982] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:28:55.983+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:28:55.983] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53368 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:28:56.020+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:28:56.020] clusterConfig unchanged [2021-10-22T01:28:56.271+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:56.271] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:56.695+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:28:56.695] ...Finished start [pid=5131] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53368,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:28:56.811+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:28:56.811] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 115.480748ms. [2021-10-22T01:28:56.811+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:28:56.811] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53368 (local=false) [2021-10-22T01:28:56.811+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:28:56.811] Dropping existing documents from system.replset [2021-10-22T01:28:56.992+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:28:56.992] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:28:57.041+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:28:57.041] Inserting oplogTruncateAfterPoint document [2021-10-22T01:28:57.041+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:28:57.041] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:28:57.043+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:28:57.043] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:28:57.043+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:28:57.043] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:28:57.057+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:28:57.057] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:28:57.057+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:28:57.057] Setting the minvalid document to [{_id ObjectID("617213d9523370002becc584")} {t -1} {ts {0 1}}] [2021-10-22T01:28:57.070+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:28:57.070] Inserted [{_id ObjectID("617213d9523370002becc584")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:28:57.070+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:28:57.070] Inserting new replica set configuration document. [2021-10-22T01:28:57.070+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:28:57.070] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:28:57.082+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:28:57.082] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:28:57.114+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:28:57.114] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:28:57.114+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:28:57.114] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:28:57.114+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:28:57.114] Initiating stopEphemeralMongo [2021-10-22T01:28:57.114+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:28:57.114] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53368 (local=false) [2021-10-22T01:28:57.114+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:28:57.114] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53368 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:28:57.139+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:28:57.139] clusterConfig unchanged [2021-10-22T01:28:57.278+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:57.278] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:57.559+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:28:57.559] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.066843ms. [2021-10-22T01:28:57.559+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:28:57.559] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53368 (local=false) [2021-10-22T01:28:57.559+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:28:57.559] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.774µs. [2021-10-22T01:28:57.559+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:28:57.559] Successfully finished running stopEphemeralMongo [2021-10-22T01:28:57.560+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:28:57.559] Successfully stopped existing ephemeral mongod. [2021-10-22T01:28:57.560+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:28:57.560] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:28:57.560+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:28:57.560] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:28:57.595+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:28:57.595] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:28:57.596+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:28:57.596] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 43973 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:28:58.272+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:58.272] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:58.275+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:28:58.275] clusterConfig unchanged [2021-10-22T01:28:58.804+0000] [.error] [src/action/start.go:func1:93] [103] [01:28:58.804] Error starting mongod : [01:28:58.804] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":43973,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 43973 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 6712 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:28:57.645+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:28:57.645+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:28:57.645+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":43973,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:28:57.647+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:28:57.649+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:28:57.659+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:28:57.688+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:28:57.688+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:28:57.688+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:28:57.742+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:28:57.742+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:28:58.294+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866138:294527][6712:0x7f5227434bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:28:58.396+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866138:396230][6712:0x7f5227434bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:28:58.497+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866138:497456][6712:0x7f5227434bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:28:58.613+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866138:613872][6712:0x7f5227434bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:28:58.694+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866138:694412][6712:0x7f5227434bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:28:58.754+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866138:754202][6712:0x7f5227434bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:28:58.754+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866138:754260][6712:0x7f5227434bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:28:58.756+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:28:58.805+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:28:58.805] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:43973 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:28:58.804] Error starting mongod : [01:28:58.804] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":43973,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 43973 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 6712 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:28:57.645+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:28:57.645+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:28:57.645+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":43973,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:28:57.647+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:28:57.649+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:28:57.659+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:28:57.688+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:28:57.688+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:28:57.688+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:28:57.742+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:28:57.742+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:28:58.294+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866138:294527][6712:0x7f5227434bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:28:58.396+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866138:396230][6712:0x7f5227434bc0], txn-recover...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:28:58.805+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:28:58.805] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:28:58.805] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:43973 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:28:58.804] Error starting mongod : [01:28:58.804] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":43973,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 43973 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 6712 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:28:57.645+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:28:57.645+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:28:57.645+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":43973,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:28:57.647+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:28:57.649+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:28:57.659+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:28:57.688+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:28:57.688+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:28:57.688+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:28:57.742+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:28:57.742+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:28:58.294+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866138:294527][6712:0x7f5227434bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:28:58.396+00:00"},"s":"I", "c":"STORAGE", "id":22430, ... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:28:58.805+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:28:58.805] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:28:58.805+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:28:58.805] Initiating stopEphemeralMongo [2021-10-22T01:28:58.805+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:28:58.805] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:28:58.807+0000] [.error] [src/director/director.go:executePlan:962] [01:28:58.807] Failed to apply action. Result = : [01:28:58.805] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:28:58.805] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:43973 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:28:58.804] Error starting mongod : [01:28:58.804] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":43973,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 43973 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 6712 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:28:57.645+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:28:57.645+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:28:57.645+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":43973,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:28:57.647+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:28:57.649+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:28:57.659+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:28:57.688+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:28:57.688+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:28:57.688+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:28:57.742+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:28:57.742+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:28:58.294+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866138:294527][6712:0x7f5227434bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:28:58.807+0000] [.error] [src/director/director.go:planAndExecute:567] [01:28:58.807] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:28:58.807] Failed to apply action. Result = : [01:28:58.805] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:28:58.805] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:43973 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:28:58.804] Error starting mongod : [01:28:58.804] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":43973,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 43973 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 6712 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:28:57.645+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:28:57.645+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:28:57.645+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":43973,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:28:57.647+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:28:57.649+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:28:57.659+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:28:57.688+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:28:57.688+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:28:57.688+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:28:57.742+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:28:57.742+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:28:58.294+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger m... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:28:58.807+0000] [.error] [src/director/director.go:mainLoop:399] [01:28:58.807] Failed to planAndExecute : [01:28:58.807] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:28:58.807] Failed to apply action. Result = : [01:28:58.805] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:28:58.805] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:43973 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:28:58.804] Error starting mongod : [01:28:58.804] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":43973,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 43973 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 6712 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:28:57.645+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:28:57.645+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:28:57.645+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":43973,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:28:57.647+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:28:57.649+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:28:57.659+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:28:57.688+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:28:57.688+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:28:57.688+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:28:57.742+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:28:57.742+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:28:58.294+00:00"},"s":"I",... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:28:59.273+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:28:59.273] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:28:59.505+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:28:59.505] clusterConfig unchanged [2021-10-22T01:29:00.274+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:00.274] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:01.272+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:01.272] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:02.274+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:02.274] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:03.273+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:03.273] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:04.281+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:04.281] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:05.277+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:05.277] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:06.278+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:06.277] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:07.278+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:07.278] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:07.675+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:29:07.675] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.568875762s [2021-10-22T01:29:08.277+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:08.277] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:09.278+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:09.278] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:09.597+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:29:09.597] clusterConfig unchanged [2021-10-22T01:29:10.279+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:10.279] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:11.279+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:11.279] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:12.281+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:12.281] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:13.282+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:13.282] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:14.282+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:14.282] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:15.288+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:15.288] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:16.287+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:16.287] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:17.283+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:17.283] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:18.284+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:18.284] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:19.284+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:19.284] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:19.739+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:29:19.739] clusterConfig unchanged [2021-10-22T01:29:20.285+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:20.285] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:21.286+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:21.286] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:22.287+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:22.287] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:23.288+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:23.288] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:24.288+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:24.288] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:25.288+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:25.288] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:26.289+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:26.289] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:27.295+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:27.295] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:28.290+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:28.290] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:29.291+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:29.291] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:29.881+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:29:29.881] clusterConfig unchanged [2021-10-22T01:29:30.291+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:30.291] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:31.292+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:31.292] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:32.293+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:32.293] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:33.293+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:33.293] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:34.293+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:34.293] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:35.294+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:35.294] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:36.295+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:36.295] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:37.295+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:37.295] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:38.296+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:38.296] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:39.296+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:39.296] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:40.022+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:29:40.022] clusterConfig unchanged [2021-10-22T01:29:40.297+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:40.297] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:41.298+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:41.298] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:41.328+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:29:41.328] memory:rss=52572160 alloc=8499528 totalalloc=5887243840 mallocs=75167771 frees=75107429 heapinuse=14761984 heapobjects=60342 gcpausetotalns=1510978406 gcpauselastns=158257 gcnum=919 utimens=112075834000 stimens=39745830000 minflt=300623 majflt=405 [2021-10-22T01:29:42.298+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:42.298] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:43.301+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:43.301] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:44.269+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [01:29:44.269] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:43973 (local=false) because it is not up [2021-10-22T01:29:44.299+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:44.299] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:45.299+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:45.299] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:46.307+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:46.306] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:47.301+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:47.301] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:48.302+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:48.302] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:49.303+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:49.303] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:50.134+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:29:50.134] clusterConfig unchanged [2021-10-22T01:29:50.303+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:50.303] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:51.304+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:51.304] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:52.304+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:52.304] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:53.305+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:53.305] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:54.305+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:54.305] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:55.306+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:55.306] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:56.307+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:56.307] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:57.307+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:57.307] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:58.308+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:58.308] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:58.977+0000] [.info] [src/director/director.go:computePlan:280] [01:29:58.977] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:29:58.977+0000] [.info] [src/director/director.go:tracef:794] [01:29:58.977] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:29:58.977+0000] [.info] [src/director/director.go:tracef:794] [01:29:58.977] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:29:58.977+0000] [.info] [src/director/director.go:tracef:794] [01:29:58.977] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:29:58.977+0000] [.info] [src/director/director.go:tracef:794] [01:29:58.977] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:29:58.977+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:29:58.977] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:29:59.126+0000] [.info] [src/director/director.go:tracef:794] [01:29:59.126] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:29:59.126+0000] [.info] [src/director/director.go:tracef:794] [01:29:59.126] because [2021-10-22T01:29:59.126+0000] [.info] [src/director/director.go:tracef:794] [01:29:59.126] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:29:59.126+0000] [.info] [src/director/director.go:tracef:794] [01:29:59.126] because ['currentState.CanBounceStop' = true] [2021-10-22T01:29:59.282+0000] [.info] [src/director/director.go:tracef:794] [01:29:59.282] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:29:59.282+0000] [.info] [src/director/director.go:tracef:794] [01:29:59.282] because [2021-10-22T01:29:59.282+0000] [.info] [src/director/director.go:tracef:794] [01:29:59.282] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:29:59.282+0000] [.info] [src/director/director.go:tracef:794] [01:29:59.282] because ['currentState.Up' = false] [2021-10-22T01:29:59.282+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:29:59.282] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:29:59.282+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:29:59.282] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:29:59.308+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:29:59.308] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:29:59.382+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:29:59.382] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:29:59.390+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:29:59.390] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:29:59.391+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:29:59.391] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:29:59.391+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:29:59.391] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:29:59.392+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:29:59.392] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:29:59.392+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:29:59.392] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:29:59.393+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:29:59.392] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:29:59.394+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:29:59.394] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:29:59.628+0000] [.info] [src/director/director.go:tracef:794] [01:29:59.628] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:29:59.628+0000] [.info] [src/director/director.go:tracef:794] [01:29:59.628] because ['currentState.Up' = false] [2021-10-22T01:29:59.846+0000] [.info] [src/director/director.go:tracef:794] [01:29:59.845] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:29:59.846+0000] [.info] [src/director/director.go:tracef:794] [01:29:59.846] because ['currentState.Up' = false] [2021-10-22T01:29:59.846+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:29:59.846] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:30:00.059+0000] [.info] [src/director/director.go:tracef:794] [01:30:00.059] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:30:00.059+0000] [.info] [src/director/director.go:tracef:794] [01:30:00.059] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:30:00.312+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:00.312] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:00.323+0000] [.info] [src/director/director.go:tracef:794] [01:30:00.323] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:30:00.323+0000] [.info] [src/director/director.go:tracef:794] [01:30:00.323] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:30:00.323+0000] [.info] [src/director/director.go:tracef:794] [01:30:00.323] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:30:00.323+0000] [.info] [src/director/director.go:tracef:794] [01:30:00.323] because ['currentState.Up' = false] [2021-10-22T01:30:00.323+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:30:00.323] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:30:00.323+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:30:00.323] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:30:00.345+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:30:00.345] clusterConfig unchanged [2021-10-22T01:30:00.357+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:30:00.357] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:30:00.358+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:30:00.358] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47136 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:30:01.135+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:30:01.135] ...Finished start [pid=25976] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47136,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:30:01.274+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:30:01.274] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 138.817502ms. [2021-10-22T01:30:01.274+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:30:01.274] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47136 (local=false) [2021-10-22T01:30:01.274+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:30:01.274] Dropping existing documents from system.replset [2021-10-22T01:30:01.316+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:01.316] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:01.445+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:30:01.445] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:30:01.450+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:30:01.449] Inserting oplogTruncateAfterPoint document [2021-10-22T01:30:01.450+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:30:01.450] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:30:01.450+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:30:01.450] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:30:01.451+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:30:01.450] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:30:01.457+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:30:01.457] clusterConfig unchanged [2021-10-22T01:30:01.465+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:30:01.465] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:30:01.465+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:30:01.465] Setting the minvalid document to [{_id ObjectID("61721419523370002becc592")} {t -1} {ts {0 1}}] [2021-10-22T01:30:01.480+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:30:01.480] Inserted [{_id ObjectID("61721419523370002becc592")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:30:01.481+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:30:01.480] Inserting new replica set configuration document. [2021-10-22T01:30:01.481+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:30:01.481] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:30:01.496+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:30:01.496] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:30:01.500+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:30:01.500] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:30:01.500+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:30:01.500] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:30:01.500+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:30:01.500] Initiating stopEphemeralMongo [2021-10-22T01:30:01.500+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:30:01.500] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47136 (local=false) [2021-10-22T01:30:01.500+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:30:01.500] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47136 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:30:01.857+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:30:01.857] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.154595ms. [2021-10-22T01:30:01.857+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:30:01.857] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47136 (local=false) [2021-10-22T01:30:01.857+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:30:01.857] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 39.313µs. [2021-10-22T01:30:01.857+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:30:01.857] Successfully finished running stopEphemeralMongo [2021-10-22T01:30:01.857+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:30:01.857] Successfully stopped existing ephemeral mongod. [2021-10-22T01:30:01.857+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:30:01.857] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:30:01.857+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:30:01.857] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:30:01.897+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:30:01.897] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:30:01.898+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:30:01.898] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36257 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:30:02.321+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:02.321] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:02.560+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:30:02.560] clusterConfig unchanged [2021-10-22T01:30:03.127+0000] [.error] [src/action/start.go:func1:93] [103] [01:30:03.127] Error starting mongod : [01:30:03.127] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36257,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36257 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:30:01.943Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 27459 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:30:01.976+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:30:01.976+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:30:01.976+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36257,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:30:01.978+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:30:01.980+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:30:01.980+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:30:01.994+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:30:01.994+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:30:01.994+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:30:02.039+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:30:02.039+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:30:02.597+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866202:597103][27459:0x7f2b1dd41bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:30:02.703+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866202:703021][27459:0x7f2b1dd41bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:30:02.808+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866202:808957][27459:0x7f2b1dd41bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:30:02.924+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866202:924701][27459:0x7f2b1dd41bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:30:03.000+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866203:870][27459:0x7f2b1dd41bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:30:03.075+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866203:75188][27459:0x7f2b1dd41bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:30:03.075+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866203:75259][27459:0x7f2b1dd41bc0], txn-recover: [WT_VERB_RECOVERY | WT_...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:30:03.127+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:30:03.127] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36257 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:30:03.127] Error starting mongod : [01:30:03.127] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36257,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36257 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:30:01.943Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 27459 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:30:01.976+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:30:01.976+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:30:01.976+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36257,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:30:01.978+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:30:01.980+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:30:01.980+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:30:01.994+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:30:01.994+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:30:01.994+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:30:02.039+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:30:02.039+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:30:02.597+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866202:597103][27459:0x7f2b1dd41bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:30:02.703+00:00"},"s...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:30:03.128+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:30:03.128] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:30:03.127] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36257 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:30:03.127] Error starting mongod : [01:30:03.127] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36257,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36257 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:30:01.943Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 27459 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:30:01.976+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:30:01.976+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:30:01.976+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36257,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:30:01.978+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:30:01.980+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:30:01.980+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:30:01.994+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:30:01.994+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:30:01.994+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:30:02.039+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:30:02.039+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:30:02.597+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866202:597103][27459:0x7f2b1dd41bc0... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:30:03.128+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:30:03.128] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:30:03.128+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:30:03.128] Initiating stopEphemeralMongo [2021-10-22T01:30:03.128+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:30:03.128] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:30:03.128+0000] [.error] [src/director/director.go:executePlan:962] [01:30:03.128] Failed to apply action. Result = : [01:30:03.128] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:30:03.127] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36257 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:30:03.127] Error starting mongod : [01:30:03.127] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36257,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36257 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:30:01.943Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 27459 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:30:01.976+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:30:01.976+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:30:01.976+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36257,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:30:01.978+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:30:01.980+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:30:01.980+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:30:01.994+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:30:01.994+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:30:01.994+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:30:02.039+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:30:02.039+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:30:02.597+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:30:03.128+0000] [.error] [src/director/director.go:planAndExecute:567] [01:30:03.128] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:30:03.128] Failed to apply action. Result = : [01:30:03.128] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:30:03.127] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36257 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:30:03.127] Error starting mongod : [01:30:03.127] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36257,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36257 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:30:01.943Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 27459 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:30:01.976+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:30:01.976+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:30:01.976+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36257,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:30:01.978+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:30:01.980+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:30:01.980+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:30:01.994+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:30:01.994+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:30:01.994+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:30:02.039+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:30:02.039+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:30:03.129+0000] [.error] [src/director/director.go:mainLoop:399] [01:30:03.129] Failed to planAndExecute : [01:30:03.128] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:30:03.128] Failed to apply action. Result = : [01:30:03.128] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:30:03.127] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36257 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:30:03.127] Error starting mongod : [01:30:03.127] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36257,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36257 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:30:01.943Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 27459 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:30:01.976+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:30:01.976+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:30:01.976+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36257,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:30:01.978+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:30:01.980+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:30:01.980+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:30:01.994+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:30:01.994+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:30:01.994+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:30:02.039+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:30:02.039+00:00"},"s":"I", "c":"NETWORK", "id":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:30:03.322+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:03.322] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:03.802+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:30:03.802] clusterConfig unchanged [2021-10-22T01:30:04.322+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:04.322] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:05.322+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:05.322] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:06.322+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:06.322] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:07.339+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:07.339] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:08.323+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:08.323] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:09.324+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:09.324] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:10.324+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:10.324] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:11.324+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:11.324] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:12.105+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:30:12.105] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.710833343s [2021-10-22T01:30:12.322+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:12.322] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:13.323+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:13.323] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:13.932+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:30:13.932] clusterConfig unchanged [2021-10-22T01:30:14.323+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:14.323] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:15.328+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:15.328] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:16.325+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:16.325] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:17.326+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:17.326] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:18.325+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:18.325] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:19.325+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:19.325] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:20.325+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:20.325] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:21.326+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:21.326] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:22.327+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:22.327] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:23.327+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:23.327] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:24.054+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:30:24.054] clusterConfig unchanged [2021-10-22T01:30:24.328+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:24.328] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:25.328+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:25.328] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:26.329+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:26.329] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:27.333+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:27.333] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:28.330+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:28.330] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:29.330+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:29.330] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:30.330+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:30.330] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:31.331+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:31.331] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:32.333+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:32.333] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:33.333+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:33.333] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:34.186+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:30:34.186] clusterConfig unchanged [2021-10-22T01:30:34.333+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:34.333] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:35.334+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:35.334] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:36.335+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:36.335] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:37.336+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:37.336] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:38.337+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:38.337] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:39.341+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:39.341] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:40.338+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:40.338] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:41.329+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:30:41.329] memory:rss=53653504 alloc=8225472 totalalloc=5975667696 mallocs=76289410 frees=76231267 heapinuse=15024128 heapobjects=58143 gcpausetotalns=1512370511 gcpauselastns=71138 gcnum=932 utimens=114038562000 stimens=40562693000 minflt=305619 majflt=405 [2021-10-22T01:30:41.338+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:41.338] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:42.338+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:42.338] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:43.338+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:43.338] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:44.317+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:30:44.317] clusterConfig unchanged [2021-10-22T01:30:44.338+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:44.338] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:45.340+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:45.340] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:46.341+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:46.341] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:47.340+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:47.340] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:48.341+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:48.341] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:49.341+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:49.341] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:50.342+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:50.342] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:51.342+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:51.342] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:52.343+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:52.343] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:53.343+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:53.343] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:54.344+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:54.344] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:54.429+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:30:54.429] clusterConfig unchanged [2021-10-22T01:30:55.345+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:55.345] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:56.346+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:56.346] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:57.347+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:57.347] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:58.353+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:58.353] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:30:59.347+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:30:59.347] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:00.348+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:00.348] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:01.350+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:01.349] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:02.349+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:02.349] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:03.299+0000] [.info] [src/director/director.go:computePlan:280] [01:31:03.299] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:31:03.299+0000] [.info] [src/director/director.go:tracef:794] [01:31:03.299] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:31:03.299+0000] [.info] [src/director/director.go:tracef:794] [01:31:03.299] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:31:03.299+0000] [.info] [src/director/director.go:tracef:794] [01:31:03.299] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:31:03.299+0000] [.info] [src/director/director.go:tracef:794] [01:31:03.299] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:31:03.299+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:31:03.299] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:31:03.365+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:03.365] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:03.475+0000] [.info] [src/director/director.go:tracef:794] [01:31:03.475] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:31:03.475+0000] [.info] [src/director/director.go:tracef:794] [01:31:03.475] because [2021-10-22T01:31:03.475+0000] [.info] [src/director/director.go:tracef:794] [01:31:03.475] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:31:03.475+0000] [.info] [src/director/director.go:tracef:794] [01:31:03.475] because ['currentState.CanBounceStop' = true] [2021-10-22T01:31:03.632+0000] [.info] [src/director/director.go:tracef:794] [01:31:03.632] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:31:03.632+0000] [.info] [src/director/director.go:tracef:794] [01:31:03.632] because [2021-10-22T01:31:03.633+0000] [.info] [src/director/director.go:tracef:794] [01:31:03.633] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:31:03.633+0000] [.info] [src/director/director.go:tracef:794] [01:31:03.633] because ['currentState.Up' = false] [2021-10-22T01:31:03.633+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:31:03.633] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:31:03.633+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:31:03.633] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:31:03.714+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:31:03.714] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:31:03.715+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:31:03.715] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:31:03.723+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:31:03.723] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:31:03.725+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:31:03.725] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:31:03.725+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:31:03.725] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:31:03.725+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:31:03.725] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:31:03.726+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:31:03.726] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:31:03.948+0000] [.info] [src/director/director.go:tracef:794] [01:31:03.948] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:31:03.948+0000] [.info] [src/director/director.go:tracef:794] [01:31:03.948] because ['currentState.Up' = false] [2021-10-22T01:31:04.179+0000] [.info] [src/director/director.go:tracef:794] [01:31:04.179] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:31:04.179+0000] [.info] [src/director/director.go:tracef:794] [01:31:04.179] because ['currentState.Up' = false] [2021-10-22T01:31:04.179+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:31:04.179] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:31:04.359+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:04.358] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:04.390+0000] [.info] [src/director/director.go:tracef:794] [01:31:04.390] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:31:04.390+0000] [.info] [src/director/director.go:tracef:794] [01:31:04.390] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:31:04.606+0000] [.info] [src/director/director.go:tracef:794] [01:31:04.606] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:31:04.606+0000] [.info] [src/director/director.go:tracef:794] [01:31:04.606] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:31:04.606+0000] [.info] [src/director/director.go:tracef:794] [01:31:04.606] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:31:04.606+0000] [.info] [src/director/director.go:tracef:794] [01:31:04.606] because ['currentState.Up' = false] [2021-10-22T01:31:04.606+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:31:04.606] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:31:04.606+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:31:04.606] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:31:04.611+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:31:04.611] clusterConfig unchanged [2021-10-22T01:31:04.643+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:31:04.643] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:31:04.643+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:31:04.643] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60103 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:31:05.353+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:05.353] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:05.367+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:31:05.367] ...Finished start [pid=46587] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60103,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:31:05.493+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:31:05.493] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 125.611354ms. [2021-10-22T01:31:05.493+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:31:05.493] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:60103 (local=false) [2021-10-22T01:31:05.493+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:31:05.493] Dropping existing documents from system.replset [2021-10-22T01:31:05.625+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:31:05.625] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:31:05.630+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:31:05.630] Inserting oplogTruncateAfterPoint document [2021-10-22T01:31:05.630+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:31:05.630] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:31:05.631+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:31:05.631] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:31:05.631+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:31:05.631] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:31:05.645+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:31:05.645] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:31:05.645+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:31:05.645] Setting the minvalid document to [{_id ObjectID("61721459523370002becc5a0")} {t -1} {ts {0 1}}] [2021-10-22T01:31:05.658+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:31:05.658] Inserted [{_id ObjectID("61721459523370002becc5a0")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:31:05.658+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:31:05.658] Inserting new replica set configuration document. [2021-10-22T01:31:05.658+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:31:05.658] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:31:05.671+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:31:05.671] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:31:05.676+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:31:05.676] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:31:05.676+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:31:05.676] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:31:05.676+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:31:05.676] Initiating stopEphemeralMongo [2021-10-22T01:31:05.676+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:31:05.676] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:60103 (local=false) [2021-10-22T01:31:05.676+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:31:05.676] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:60103 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:31:05.733+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:31:05.733] clusterConfig unchanged [2021-10-22T01:31:06.030+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:31:06.030] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.18723ms. [2021-10-22T01:31:06.030+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:31:06.030] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:60103 (local=false) [2021-10-22T01:31:06.030+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:31:06.030] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 39.154µs. [2021-10-22T01:31:06.030+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:31:06.030] Successfully finished running stopEphemeralMongo [2021-10-22T01:31:06.030+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:31:06.030] Successfully stopped existing ephemeral mongod. [2021-10-22T01:31:06.030+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:31:06.030] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:31:06.030+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:31:06.030] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:31:06.065+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:31:06.064] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:31:06.065+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:31:06.065] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60718 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:31:06.354+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:06.354] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:06.855+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:31:06.855] clusterConfig unchanged [2021-10-22T01:31:07.289+0000] [.error] [src/action/start.go:func1:93] [103] [01:31:07.289] Error starting mongod : [01:31:07.288] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60718 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:31:06.110Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 48135 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:31:06.128+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:31:06.128+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:31:06.128+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:31:06.129+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:31:06.131+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:31:06.132+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:31:06.143+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:31:06.143+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:31:06.144+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:31:06.191+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:31:06.191+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:31:06.729+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866266:729211][48135:0x7f64dbccbbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:31:06.845+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866266:845098][48135:0x7f64dbccbbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:31:06.949+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866266:949268][48135:0x7f64dbccbbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35456 to 2/256"}} {"t":{"$date":"2021-10-22T01:31:07.065+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866267:65574][48135:0x7f64dbccbbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:31:07.161+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866267:161164][48135:0x7f64dbccbbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:31:07.235+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866267:235191][48135:0x7f64dbccbbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:31:07.235+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866267:235275][48135:0x7f64dbccbbc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:31:07.290+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:31:07.290] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60718 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:31:07.289] Error starting mongod : [01:31:07.288] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60718 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:31:06.110Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 48135 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:31:06.128+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:31:06.128+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:31:06.128+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:31:06.129+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:31:06.131+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:31:06.132+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:31:06.143+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:31:06.143+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:31:06.144+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:31:06.191+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:31:06.191+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:31:06.729+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866266:729211][48135:0x7f64dbccbbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:31:06.845+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:31:07.291+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:31:07.291] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:31:07.290] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60718 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:31:07.289] Error starting mongod : [01:31:07.288] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60718 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:31:06.110Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 48135 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:31:06.128+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:31:06.128+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:31:06.128+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:31:06.129+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:31:06.131+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:31:06.132+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:31:06.143+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:31:06.143+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:31:06.144+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:31:06.191+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:31:06.191+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:31:06.729+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866266:729211][48135:0x7f64dbccbbc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:31:07.291+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:31:07.291] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:31:07.292+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:31:07.291] Initiating stopEphemeralMongo [2021-10-22T01:31:07.292+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:31:07.292] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:31:07.299+0000] [.error] [src/director/director.go:executePlan:962] [01:31:07.299] Failed to apply action. Result = : [01:31:07.291] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:31:07.290] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60718 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:31:07.289] Error starting mongod : [01:31:07.288] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60718 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:31:06.110Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 48135 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:31:06.128+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:31:06.128+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:31:06.128+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:31:06.129+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:31:06.131+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:31:06.132+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:31:06.143+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:31:06.143+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:31:06.144+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:31:06.191+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:31:06.191+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:31:06.729+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:31:07.299+0000] [.error] [src/director/director.go:planAndExecute:567] [01:31:07.299] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:31:07.299] Failed to apply action. Result = : [01:31:07.291] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:31:07.290] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60718 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:31:07.289] Error starting mongod : [01:31:07.288] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60718 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:31:06.110Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 48135 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:31:06.128+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:31:06.128+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:31:06.128+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:31:06.129+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:31:06.131+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:31:06.132+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:31:06.143+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:31:06.143+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:31:06.144+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:31:06.191+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:31:06.191+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:31:07.299+0000] [.error] [src/director/director.go:mainLoop:399] [01:31:07.299] Failed to planAndExecute : [01:31:07.299] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:31:07.299] Failed to apply action. Result = : [01:31:07.291] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:31:07.290] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60718 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:31:07.289] Error starting mongod : [01:31:07.288] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60718 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:31:06.110Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 48135 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:31:06.128+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:31:06.128+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:31:06.128+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:31:06.129+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:31:06.131+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:31:06.132+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:31:06.143+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:31:06.143+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:31:06.144+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:31:06.191+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:31:06.191+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:31:07.354+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:07.354] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:08.194+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:31:08.194] clusterConfig unchanged [2021-10-22T01:31:08.355+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:08.355] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:09.355+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:09.354] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:10.354+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:10.354] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:11.356+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:11.356] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:12.354+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:12.354] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:13.358+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:13.358] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:14.356+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:14.356] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:15.357+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:15.357] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:16.327+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:31:16.327] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.600467822s [2021-10-22T01:31:16.355+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:16.354] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:17.356+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:17.356] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:18.316+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:31:18.316] clusterConfig unchanged [2021-10-22T01:31:18.356+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:18.356] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:18.560+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:31:18.560] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:31:18.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:18.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:31:18.562+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [01:31:18.562] Could not get version : [01:31:18.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:31:18.562+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [01:31:18.562] Error getting version for log lines. Skipping : [01:31:18.562] Could not get version : [01:31:18.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:31:19.356+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:19.356] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:20.357+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:20.357] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:21.357+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:21.357] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:22.362+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:22.362] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:23.358+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:23.358] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:24.359+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:24.359] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:25.360+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:25.360] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:26.360+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:26.360] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:27.361+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:27.361] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:28.361+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:28.361] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:28.438+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:31:28.438] clusterConfig unchanged [2021-10-22T01:31:29.362+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:29.362] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:30.362+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:30.362] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:31.363+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:31.363] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:32.364+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:32.364] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:33.364+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:33.364] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:34.370+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:34.370] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:35.366+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:35.366] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:36.366+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:36.366] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:37.372+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:37.372] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:38.367+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:38.367] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:38.540+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:31:38.540] clusterConfig unchanged [2021-10-22T01:31:39.367+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:39.367] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:40.367+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:40.367] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:41.329+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:31:41.329] memory:rss=53059584 alloc=10951920 totalalloc=6065022856 mallocs=77414084 frees=77326859 heapinuse=15736832 heapobjects=87225 gcpausetotalns=1513607987 gcpauselastns=53649 gcnum=944 utimens=115920561000 stimens=41278161000 minflt=310506 majflt=405 [2021-10-22T01:31:41.372+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:41.372] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:42.368+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:42.368] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:43.369+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:43.369] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:44.370+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:44.370] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:45.371+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:45.371] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:46.371+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:46.371] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:47.372+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:47.372] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:48.372+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:48.372] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:48.683+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:31:48.683] clusterConfig unchanged [2021-10-22T01:31:49.373+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:49.373] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:50.374+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:50.374] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:51.375+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:51.375] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:52.375+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:52.375] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:53.378+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:53.378] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:54.377+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:54.377] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:55.378+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:55.378] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:56.378+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:56.378] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:57.379+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:57.379] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:58.379+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:58.379] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:31:58.815+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:31:58.815] clusterConfig unchanged [2021-10-22T01:31:59.380+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:31:59.380] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:00.381+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:00.381] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:01.384+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:01.384] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:02.383+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:02.382] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:03.384+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:03.384] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:04.390+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:04.390] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:05.384+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:05.384] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:06.386+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:06.386] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:07.388+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:07.388] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:07.463+0000] [.info] [src/director/director.go:computePlan:280] [01:32:07.463] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:32:07.463+0000] [.info] [src/director/director.go:tracef:794] [01:32:07.463] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:32:07.463+0000] [.info] [src/director/director.go:tracef:794] [01:32:07.463] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:32:07.464+0000] [.info] [src/director/director.go:tracef:794] [01:32:07.464] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:32:07.464+0000] [.info] [src/director/director.go:tracef:794] [01:32:07.464] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:32:07.464+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:32:07.464] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:32:07.625+0000] [.info] [src/director/director.go:tracef:794] [01:32:07.625] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:32:07.625+0000] [.info] [src/director/director.go:tracef:794] [01:32:07.625] because [2021-10-22T01:32:07.626+0000] [.info] [src/director/director.go:tracef:794] [01:32:07.626] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:32:07.626+0000] [.info] [src/director/director.go:tracef:794] [01:32:07.626] because ['currentState.CanBounceStop' = true] [2021-10-22T01:32:07.798+0000] [.info] [src/director/director.go:tracef:794] [01:32:07.798] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:32:07.798+0000] [.info] [src/director/director.go:tracef:794] [01:32:07.798] because [2021-10-22T01:32:07.798+0000] [.info] [src/director/director.go:tracef:794] [01:32:07.798] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:32:07.798+0000] [.info] [src/director/director.go:tracef:794] [01:32:07.798] because ['currentState.Up' = false] [2021-10-22T01:32:07.798+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:32:07.798] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:32:07.798+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:32:07.798] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:32:07.885+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:32:07.885] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:32:07.896+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:32:07.896] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:32:07.896+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:32:07.896] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:32:07.897+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:32:07.897] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:32:07.898+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:32:07.898] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:32:07.898+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:32:07.898] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:32:07.898+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:32:07.898] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:32:07.899+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:32:07.899] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:32:08.122+0000] [.info] [src/director/director.go:tracef:794] [01:32:08.122] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:32:08.122+0000] [.info] [src/director/director.go:tracef:794] [01:32:08.122] because ['currentState.Up' = false] [2021-10-22T01:32:08.339+0000] [.info] [src/director/director.go:tracef:794] [01:32:08.339] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:32:08.339+0000] [.info] [src/director/director.go:tracef:794] [01:32:08.339] because ['currentState.Up' = false] [2021-10-22T01:32:08.339+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:32:08.339] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:32:08.390+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:08.390] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:08.563+0000] [.info] [src/director/director.go:tracef:794] [01:32:08.563] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:32:08.563+0000] [.info] [src/director/director.go:tracef:794] [01:32:08.563] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:32:08.776+0000] [.info] [src/director/director.go:tracef:794] [01:32:08.776] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:32:08.776+0000] [.info] [src/director/director.go:tracef:794] [01:32:08.776] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:32:08.776+0000] [.info] [src/director/director.go:tracef:794] [01:32:08.776] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:32:08.776+0000] [.info] [src/director/director.go:tracef:794] [01:32:08.776] because ['currentState.Up' = false] [2021-10-22T01:32:08.776+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:32:08.776] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:32:08.776+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:32:08.776] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:32:08.810+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:32:08.810] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:32:08.810+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:32:08.810] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41975 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:32:09.036+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:32:09.036] clusterConfig unchanged [2021-10-22T01:32:09.390+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:09.390] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:09.533+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:32:09.533] ...Finished start [pid=67414] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41975,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:32:09.679+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:32:09.679] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 146.123777ms. [2021-10-22T01:32:09.679+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:32:09.679] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41975 (local=false) [2021-10-22T01:32:09.679+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:32:09.679] Dropping existing documents from system.replset [2021-10-22T01:32:09.822+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:32:09.822] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:32:09.826+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:32:09.826] Inserting oplogTruncateAfterPoint document [2021-10-22T01:32:09.826+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:32:09.826] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:32:09.827+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:32:09.827] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:32:09.827+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:32:09.827] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:32:09.839+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:32:09.839] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:32:09.839+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:32:09.839] Setting the minvalid document to [{_id ObjectID("61721499523370002becc5ae")} {t -1} {ts {0 1}}] [2021-10-22T01:32:09.853+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:32:09.853] Inserted [{_id ObjectID("61721499523370002becc5ae")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:32:09.853+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:32:09.853] Inserting new replica set configuration document. [2021-10-22T01:32:09.853+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:32:09.853] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:32:09.864+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:32:09.864] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:32:09.898+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:32:09.898] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:32:09.898+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:32:09.898] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:32:09.898+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:32:09.898] Initiating stopEphemeralMongo [2021-10-22T01:32:09.898+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:32:09.898] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41975 (local=false) [2021-10-22T01:32:09.898+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:32:09.898] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41975 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:32:10.151+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:32:10.151] clusterConfig unchanged [2021-10-22T01:32:10.318+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:32:10.318] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.551861ms. [2021-10-22T01:32:10.318+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:32:10.318] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41975 (local=false) [2021-10-22T01:32:10.319+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:32:10.318] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 49.835µs. [2021-10-22T01:32:10.319+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:32:10.319] Successfully finished running stopEphemeralMongo [2021-10-22T01:32:10.319+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:32:10.319] Successfully stopped existing ephemeral mongod. [2021-10-22T01:32:10.319+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:32:10.319] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:32:10.319+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:32:10.319] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:32:10.353+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:32:10.352] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:32:10.353+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:32:10.353] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33145 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:32:10.390+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:10.390] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:11.260+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:32:11.260] clusterConfig unchanged [2021-10-22T01:32:11.398+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:11.398] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:11.617+0000] [.error] [src/action/start.go:func1:93] [103] [01:32:11.617] Error starting mongod : [01:32:11.617] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33145,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33145 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:32:10.382Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 68833 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:32:10.404+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:32:10.404+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:32:10.404+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33145,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:32:10.406+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:32:10.410+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:32:10.415+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:32:10.442+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:32:10.442+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:32:10.444+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:32:10.495+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:32:10.495+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:32:11.014+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866331:14849][68833:0x7efdfd83fbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:32:11.121+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866331:121484][68833:0x7efdfd83fbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:32:11.221+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866331:221488][68833:0x7efdfd83fbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:32:11.333+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866331:333182][68833:0x7efdfd83fbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:32:11.476+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866331:476611][68833:0x7efdfd83fbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:32:11.545+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866331:545068][68833:0x7efdfd83fbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:32:11.545+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866331:545140][68833:0x7efdfd83fbc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:32:11.617+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:32:11.617] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33145 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:32:11.617] Error starting mongod : [01:32:11.617] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33145,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33145 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:32:10.382Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 68833 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:32:10.404+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:32:10.404+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:32:10.404+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33145,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:32:10.406+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:32:10.410+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:32:10.415+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:32:10.442+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:32:10.442+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:32:10.444+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:32:10.495+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:32:10.495+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:32:11.014+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866331:14849][68833:0x7efdfd83fbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:32:11.121+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:32:11.617+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:32:11.617] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:32:11.617] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33145 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:32:11.617] Error starting mongod : [01:32:11.617] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33145,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33145 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:32:10.382Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 68833 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:32:10.404+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:32:10.404+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:32:10.404+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33145,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:32:10.406+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:32:10.410+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:32:10.415+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:32:10.442+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:32:10.442+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:32:10.444+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:32:10.495+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:32:10.495+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:32:11.014+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866331:14849][68833:0x7efdfd83fbc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:32:11.617+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:32:11.617] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:32:11.617+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:32:11.617] Initiating stopEphemeralMongo [2021-10-22T01:32:11.618+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:32:11.617] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:32:11.618+0000] [.error] [src/director/director.go:executePlan:962] [01:32:11.618] Failed to apply action. Result = : [01:32:11.617] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:32:11.617] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33145 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:32:11.617] Error starting mongod : [01:32:11.617] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33145,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33145 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:32:10.382Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 68833 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:32:10.404+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:32:10.404+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:32:10.404+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33145,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:32:10.406+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:32:10.410+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:32:10.415+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:32:10.442+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:32:10.442+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:32:10.444+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:32:10.495+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:32:10.495+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:32:11.014+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:32:11.618+0000] [.error] [src/director/director.go:planAndExecute:567] [01:32:11.618] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:32:11.618] Failed to apply action. Result = : [01:32:11.617] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:32:11.617] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33145 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:32:11.617] Error starting mongod : [01:32:11.617] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33145,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33145 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:32:10.382Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 68833 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:32:10.404+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:32:10.404+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:32:10.404+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33145,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:32:10.406+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:32:10.410+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:32:10.415+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:32:10.442+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:32:10.442+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:32:10.444+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:32:10.495+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:32:10.495+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:32:11.618+0000] [.error] [src/director/director.go:mainLoop:399] [01:32:11.618] Failed to planAndExecute : [01:32:11.618] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:32:11.618] Failed to apply action. Result = : [01:32:11.617] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:32:11.617] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33145 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:32:11.617] Error starting mongod : [01:32:11.617] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33145,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33145 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:32:10.382Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 68833 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:32:10.404+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:32:10.404+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:32:10.404+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33145,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:32:10.406+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:32:10.410+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:32:10.415+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:32:10.442+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:32:10.442+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:32:10.444+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:32:10.495+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:32:10.495+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:32:12.390+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:12.390] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:12.540+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:32:12.540] clusterConfig unchanged [2021-10-22T01:32:13.392+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:13.392] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:14.394+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:14.394] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:15.391+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:15.391] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:16.394+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:16.394] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:17.394+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:17.394] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:18.395+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:18.395] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:19.395+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:19.395] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:20.384+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:32:20.384] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.48469986s [2021-10-22T01:32:20.394+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:20.394] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:21.394+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:21.394] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:22.400+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:22.400] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:22.652+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:32:22.652] clusterConfig unchanged [2021-10-22T01:32:23.395+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:23.395] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:24.395+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:24.395] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:25.396+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:25.396] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:26.397+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:26.397] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:27.397+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:27.397] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:27.555+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:32:27.555] ClientMetadata Mspooler loop timeout after: 1h0m0s. Flushing to mms [2021-10-22T01:32:27.556+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:27.556] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:32:27.557+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [01:32:27.556] Could not get version : [01:32:27.556] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:32:27.557+0000] [.warn] [filebeat/mspooler/clientmetadata.go:PreFlushProcessing:192] [01:32:27.557] Error getting version for metadata entry. Skipping : [01:32:27.556] Could not get version : [01:32:27.556] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:32:27.558+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:27.558] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:32:27.558+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [01:32:27.558] Could not get version : [01:32:27.558] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:32:27.558+0000] [.warn] [filebeat/mspooler/clientmetadata.go:PreFlushProcessing:192] [01:32:27.558] Error getting version for metadata entry. Skipping : [01:32:27.558] Could not get version : [01:32:27.558] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:32:27.560+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:27.560] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:32:27.560+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [01:32:27.560] Could not get version : [01:32:27.560] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:32:27.560+0000] [.warn] [filebeat/mspooler/clientmetadata.go:PreFlushProcessing:192] [01:32:27.560] Error getting version for metadata entry. Skipping : [01:32:27.560] Could not get version : [01:32:27.560] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:32:27.561+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:27.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:32:27.561+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [01:32:27.561] Could not get version : [01:32:27.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:32:27.561+0000] [.warn] [filebeat/mspooler/clientmetadata.go:PreFlushProcessing:192] [01:32:27.561] Error getting version for metadata entry. Skipping : [01:32:27.561] Could not get version : [01:32:27.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:32:27.564+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:32:27.564] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:32:27.565+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:32:27.565] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:32:27.567+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:27.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:32:27.567+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [01:32:27.567] Could not get version : [01:32:27.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:32:27.567+0000] [.warn] [filebeat/mspooler/clientmetadata.go:PreFlushProcessing:192] [01:32:27.567] Error getting version for metadata entry. Skipping : [01:32:27.567] Could not get version : [01:32:27.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:32:28.397+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:28.397] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:29.398+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:29.398] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:30.398+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:30.398] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:31.399+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:31.399] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:32.399+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:32.399] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:32.795+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:32:32.795] clusterConfig unchanged [2021-10-22T01:32:33.400+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:33.400] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:34.404+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:34.404] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:35.401+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:35.401] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:36.403+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [01:32:36.403] lsblk cache invalidated, running lsblk command [2021-10-22T01:32:36.404+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:36.404] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:37.402+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:37.402] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:38.403+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:38.403] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:39.409+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:39.409] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:40.410+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:40.410] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:41.330+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:32:41.330] memory:rss=52813824 alloc=7735104 totalalloc=6158426240 mallocs=78549208 frees=78497523 heapinuse=15294464 heapobjects=51685 gcpausetotalns=1515909804 gcpauselastns=75108 gcnum=958 utimens=117982372000 stimens=42169299000 minflt=316349 majflt=405 [2021-10-22T01:32:41.406+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:41.406] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:42.407+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:42.407] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:42.925+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:32:42.925] clusterConfig unchanged [2021-10-22T01:32:43.407+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:43.407] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:44.278+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [01:32:44.278] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:33145 (local=false) because it is not up [2021-10-22T01:32:44.408+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:44.408] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:45.415+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:45.415] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:46.408+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:46.408] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:47.409+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:47.409] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:48.415+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:48.415] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:49.409+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:49.409] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:50.409+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:50.409] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:51.409+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:51.409] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:52.409+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:52.409] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:53.067+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:32:53.067] clusterConfig unchanged [2021-10-22T01:32:53.409+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:53.409] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:54.410+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:54.410] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:55.410+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:55.410] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:56.411+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:56.411] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:57.412+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:57.412] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:58.412+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:58.412] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:32:59.418+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:32:59.418] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:00.413+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:00.412] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:01.436+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:01.436] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:02.415+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:02.415] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:03.159+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:33:03.159] clusterConfig unchanged [2021-10-22T01:33:03.415+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:03.415] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:04.416+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:04.416] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:05.416+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:05.416] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:06.417+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:06.417] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:07.418+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:07.418] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:08.419+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:08.419] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:09.419+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:09.419] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:10.426+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:10.426] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:11.420+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:11.420] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:11.792+0000] [.info] [src/director/director.go:computePlan:280] [01:33:11.792] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:33:11.792+0000] [.info] [src/director/director.go:tracef:794] [01:33:11.792] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:33:11.792+0000] [.info] [src/director/director.go:tracef:794] [01:33:11.792] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:33:11.792+0000] [.info] [src/director/director.go:tracef:794] [01:33:11.792] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:33:11.792+0000] [.info] [src/director/director.go:tracef:794] [01:33:11.792] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:33:11.792+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:33:11.792] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:33:11.943+0000] [.info] [src/director/director.go:tracef:794] [01:33:11.943] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:33:11.943+0000] [.info] [src/director/director.go:tracef:794] [01:33:11.943] because [2021-10-22T01:33:11.944+0000] [.info] [src/director/director.go:tracef:794] [01:33:11.944] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:33:11.944+0000] [.info] [src/director/director.go:tracef:794] [01:33:11.944] because ['currentState.CanBounceStop' = true] [2021-10-22T01:33:12.102+0000] [.info] [src/director/director.go:tracef:794] [01:33:12.102] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:33:12.102+0000] [.info] [src/director/director.go:tracef:794] [01:33:12.102] because [2021-10-22T01:33:12.102+0000] [.info] [src/director/director.go:tracef:794] [01:33:12.102] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:33:12.102+0000] [.info] [src/director/director.go:tracef:794] [01:33:12.102] because ['currentState.Up' = false] [2021-10-22T01:33:12.102+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:33:12.102] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:33:12.102+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:33:12.102] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:33:12.188+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:33:12.188] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:33:12.197+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:33:12.197] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:33:12.197+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:33:12.197] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:33:12.198+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:33:12.197] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:33:12.199+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:33:12.199] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:33:12.199+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:33:12.199] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:33:12.199+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:33:12.199] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:33:12.201+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:33:12.201] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:33:12.426+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:12.426] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:12.428+0000] [.info] [src/director/director.go:tracef:794] [01:33:12.428] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:33:12.428+0000] [.info] [src/director/director.go:tracef:794] [01:33:12.428] because ['currentState.Up' = false] [2021-10-22T01:33:12.615+0000] [.info] [src/director/director.go:tracef:794] [01:33:12.615] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:33:12.615+0000] [.info] [src/director/director.go:tracef:794] [01:33:12.615] because ['currentState.Up' = false] [2021-10-22T01:33:12.615+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:33:12.615] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:33:12.816+0000] [.info] [src/director/director.go:tracef:794] [01:33:12.816] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:33:12.816+0000] [.info] [src/director/director.go:tracef:794] [01:33:12.816] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:33:13.049+0000] [.info] [src/director/director.go:tracef:794] [01:33:13.049] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:33:13.049+0000] [.info] [src/director/director.go:tracef:794] [01:33:13.049] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:33:13.049+0000] [.info] [src/director/director.go:tracef:794] [01:33:13.049] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:33:13.049+0000] [.info] [src/director/director.go:tracef:794] [01:33:13.049] because ['currentState.Up' = false] [2021-10-22T01:33:13.049+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:33:13.049] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:33:13.049+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:33:13.049] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:33:13.081+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:33:13.081] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:33:13.082+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:33:13.082] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33108 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:33:13.351+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:33:13.351] clusterConfig unchanged [2021-10-22T01:33:13.438+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:13.438] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:13.796+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:33:13.796] ...Finished start [pid=14161] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33108,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:33:13.926+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:33:13.926] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 130.313435ms. [2021-10-22T01:33:13.926+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:33:13.926] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:33108 (local=false) [2021-10-22T01:33:13.926+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:33:13.926] Dropping existing documents from system.replset [2021-10-22T01:33:14.039+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:33:14.039] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:33:14.047+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:33:14.047] Inserting oplogTruncateAfterPoint document [2021-10-22T01:33:14.047+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:33:14.047] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:33:14.048+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:33:14.048] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:33:14.048+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:33:14.048] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:33:14.062+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:33:14.062] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:33:14.062+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:33:14.062] Setting the minvalid document to [{_id ObjectID("617214da523370002becc5bc")} {t -1} {ts {0 1}}] [2021-10-22T01:33:14.075+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:33:14.075] Inserted [{_id ObjectID("617214da523370002becc5bc")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:33:14.075+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:33:14.075] Inserting new replica set configuration document. [2021-10-22T01:33:14.075+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:33:14.075] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:33:14.089+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:33:14.089] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:33:14.093+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:33:14.093] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:33:14.093+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:33:14.093] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:33:14.093+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:33:14.093] Initiating stopEphemeralMongo [2021-10-22T01:33:14.093+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:33:14.093] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:33108 (local=false) [2021-10-22T01:33:14.093+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:33:14.093] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:33108 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:33:14.426+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:14.426] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:14.438+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:33:14.438] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.629808ms. [2021-10-22T01:33:14.438+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:33:14.438] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:33108 (local=false) [2021-10-22T01:33:14.438+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:33:14.438] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.164µs. [2021-10-22T01:33:14.438+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:33:14.438] Successfully finished running stopEphemeralMongo [2021-10-22T01:33:14.438+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:33:14.438] Successfully stopped existing ephemeral mongod. [2021-10-22T01:33:14.438+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:33:14.438] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:33:14.438+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:33:14.438] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:33:14.464+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:33:14.464] clusterConfig unchanged [2021-10-22T01:33:14.471+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:33:14.471] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:33:14.472+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:33:14.472] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50885 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:33:15.429+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:15.429] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:15.596+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:33:15.596] clusterConfig unchanged [2021-10-22T01:33:15.651+0000] [.error] [src/action/start.go:func1:93] [103] [01:33:15.651] Error starting mongod : [01:33:15.651] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50885 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:33:14.505Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 15606 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:33:14.525+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:33:14.525+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:33:14.525+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:33:14.527+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:33:14.529+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:33:14.530+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:33:14.578+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:33:14.578+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:33:14.579+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:33:14.626+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:33:14.626+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:33:15.135+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866395:135605][15606:0x7fdee8f9cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:33:15.242+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866395:242179][15606:0x7fdee8f9cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:33:15.342+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866395:342349][15606:0x7fdee8f9cbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T01:33:15.454+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866395:454016][15606:0x7fdee8f9cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:33:15.525+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866395:525788][15606:0x7fdee8f9cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:33:15.599+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866395:599805][15606:0x7fdee8f9cbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:33:15.599+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866395:599884][15606:0x7fdee8f9cbc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:33:15.651+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:33:15.651] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:50885 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:33:15.651] Error starting mongod : [01:33:15.651] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50885 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:33:14.505Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 15606 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:33:14.525+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:33:14.525+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:33:14.525+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:33:14.527+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:33:14.529+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:33:14.530+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:33:14.578+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:33:14.578+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:33:14.579+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:33:14.626+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:33:14.626+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:33:15.135+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866395:135605][15606:0x7fdee8f9cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:33:15.242+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:33:15.652+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:33:15.652] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:33:15.651] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:50885 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:33:15.651] Error starting mongod : [01:33:15.651] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50885 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:33:14.505Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 15606 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:33:14.525+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:33:14.525+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:33:14.525+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:33:14.527+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:33:14.529+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:33:14.530+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:33:14.578+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:33:14.578+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:33:14.579+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:33:14.626+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:33:14.626+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:33:15.135+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866395:135605][15606:0x7fdee8f9cbc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:33:15.652+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:33:15.652] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:33:15.652+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:33:15.652] Initiating stopEphemeralMongo [2021-10-22T01:33:15.652+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:33:15.652] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:33:15.655+0000] [.error] [src/director/director.go:executePlan:962] [01:33:15.655] Failed to apply action. Result = : [01:33:15.652] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:33:15.651] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:50885 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:33:15.651] Error starting mongod : [01:33:15.651] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50885 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:33:14.505Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 15606 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:33:14.525+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:33:14.525+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:33:14.525+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:33:14.527+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:33:14.529+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:33:14.530+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:33:14.578+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:33:14.578+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:33:14.579+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:33:14.626+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:33:14.626+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:33:15.135+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:33:15.656+0000] [.error] [src/director/director.go:planAndExecute:567] [01:33:15.655] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:33:15.655] Failed to apply action. Result = : [01:33:15.652] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:33:15.651] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:50885 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:33:15.651] Error starting mongod : [01:33:15.651] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50885 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:33:14.505Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 15606 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:33:14.525+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:33:14.525+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:33:14.525+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:33:14.527+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:33:14.529+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:33:14.530+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:33:14.578+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:33:14.578+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:33:14.579+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:33:14.626+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:33:14.626+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:33:15.657+0000] [.error] [src/director/director.go:mainLoop:399] [01:33:15.656] Failed to planAndExecute : [01:33:15.655] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:33:15.655] Failed to apply action. Result = : [01:33:15.652] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:33:15.651] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:50885 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:33:15.651] Error starting mongod : [01:33:15.651] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50885 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:33:14.505Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 15606 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:33:14.525+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:33:14.525+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:33:14.525+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:33:14.527+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:33:14.529+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:33:14.530+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:33:14.578+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:33:14.578+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:33:14.579+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:33:14.626+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:33:14.626+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:33:16.428+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:16.428] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:16.856+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:33:16.856] clusterConfig unchanged [2021-10-22T01:33:17.427+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:17.427] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:18.430+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:18.430] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:19.429+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:19.429] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:20.428+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:20.428] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:21.431+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:21.431] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:22.433+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:22.433] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:23.432+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:23.432] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:24.430+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:24.430] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:24.671+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:33:24.671] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.470480604s [2021-10-22T01:33:25.433+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:25.433] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:26.438+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:26.438] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:26.967+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:33:26.967] clusterConfig unchanged [2021-10-22T01:33:27.432+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:27.432] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:28.434+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:28.434] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:29.435+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:29.435] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:30.436+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:30.436] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:31.434+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:31.434] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:32.433+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:32.433] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:33.436+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:33.436] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:34.444+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:34.444] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:35.437+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:35.437] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:36.437+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:36.437] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:37.099+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:33:37.099] clusterConfig unchanged [2021-10-22T01:33:37.437+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:37.437] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:38.437+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:38.437] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:39.439+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:39.439] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:40.439+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:40.439] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:41.330+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:33:41.330] memory:rss=54636544 alloc=11263760 totalalloc=6247814488 mallocs=79684707 frees=79594787 heapinuse=16228352 heapobjects=89920 gcpausetotalns=1517502825 gcpauselastns=134158 gcnum=970 utimens=120198506000 stimens=43163568000 minflt=321451 majflt=405 [2021-10-22T01:33:41.441+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:41.441] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:42.440+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:42.440] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:43.442+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:43.442] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:44.455+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:44.455] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:45.441+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:45.441] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:46.442+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:46.442] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:47.211+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:33:47.211] clusterConfig unchanged [2021-10-22T01:33:47.442+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:47.442] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:48.442+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:48.442] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:49.442+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:49.442] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:50.442+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:50.442] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:51.443+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:51.443] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:52.443+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:52.443] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:53.444+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:53.444] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:54.444+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:54.444] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:55.444+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:55.444] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:56.454+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:56.454] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:57.313+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:33:57.313] clusterConfig unchanged [2021-10-22T01:33:57.446+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:57.446] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:58.446+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:58.446] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:33:59.447+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:33:59.447] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:00.447+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:00.447] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:01.447+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:01.447] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:02.447+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:02.447] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:03.447+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:03.447] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:04.448+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:04.448] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:05.449+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:05.449] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:06.449+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:06.449] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:07.439+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:34:07.439] clusterConfig unchanged [2021-10-22T01:34:07.456+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:07.456] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:08.452+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:08.452] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:09.451+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:09.451] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:10.451+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:10.451] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:11.452+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:11.452] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:12.452+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:12.452] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:13.453+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:13.453] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:14.453+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:14.453] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:15.455+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:15.455] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:15.819+0000] [.info] [src/director/director.go:computePlan:280] [01:34:15.819] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:34:15.819+0000] [.info] [src/director/director.go:tracef:794] [01:34:15.819] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:34:15.819+0000] [.info] [src/director/director.go:tracef:794] [01:34:15.819] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:34:15.819+0000] [.info] [src/director/director.go:tracef:794] [01:34:15.819] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:34:15.819+0000] [.info] [src/director/director.go:tracef:794] [01:34:15.819] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:34:15.819+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:34:15.819] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:34:15.975+0000] [.info] [src/director/director.go:tracef:794] [01:34:15.975] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:34:15.975+0000] [.info] [src/director/director.go:tracef:794] [01:34:15.975] because [2021-10-22T01:34:15.975+0000] [.info] [src/director/director.go:tracef:794] [01:34:15.975] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:34:15.975+0000] [.info] [src/director/director.go:tracef:794] [01:34:15.975] because ['currentState.CanBounceStop' = true] [2021-10-22T01:34:16.137+0000] [.info] [src/director/director.go:tracef:794] [01:34:16.137] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:34:16.137+0000] [.info] [src/director/director.go:tracef:794] [01:34:16.137] because [2021-10-22T01:34:16.138+0000] [.info] [src/director/director.go:tracef:794] [01:34:16.138] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:34:16.138+0000] [.info] [src/director/director.go:tracef:794] [01:34:16.138] because ['currentState.Up' = false] [2021-10-22T01:34:16.138+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:34:16.138] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:34:16.138+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:34:16.138] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:34:16.218+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:34:16.218] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:34:16.226+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:34:16.226] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:34:16.227+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:34:16.227] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:34:16.227+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:34:16.227] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:34:16.228+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:34:16.228] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:34:16.228+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:34:16.228] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:34:16.228+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:34:16.228] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:34:16.230+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:34:16.230] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:34:16.439+0000] [.info] [src/director/director.go:tracef:794] [01:34:16.439] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:34:16.439+0000] [.info] [src/director/director.go:tracef:794] [01:34:16.439] because ['currentState.Up' = false] [2021-10-22T01:34:16.494+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:16.494] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:16.662+0000] [.info] [src/director/director.go:tracef:794] [01:34:16.662] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:34:16.663+0000] [.info] [src/director/director.go:tracef:794] [01:34:16.663] because ['currentState.Up' = false] [2021-10-22T01:34:16.663+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:34:16.663] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:34:16.875+0000] [.info] [src/director/director.go:tracef:794] [01:34:16.875] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:34:16.875+0000] [.info] [src/director/director.go:tracef:794] [01:34:16.875] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:34:17.065+0000] [.info] [src/director/director.go:tracef:794] [01:34:17.065] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:34:17.065+0000] [.info] [src/director/director.go:tracef:794] [01:34:17.065] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:34:17.065+0000] [.info] [src/director/director.go:tracef:794] [01:34:17.065] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:34:17.065+0000] [.info] [src/director/director.go:tracef:794] [01:34:17.065] because ['currentState.Up' = false] [2021-10-22T01:34:17.066+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:34:17.065] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:34:17.066+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:34:17.066] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:34:17.103+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:34:17.103] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:34:17.104+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:34:17.104] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52825 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:34:17.460+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:17.460] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:17.650+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:34:17.650] clusterConfig unchanged [2021-10-22T01:34:17.831+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:34:17.831] ...Finished start [pid=35395] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52825,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:34:17.947+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:34:17.947] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 115.961988ms. [2021-10-22T01:34:17.947+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:34:17.947] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52825 (local=false) [2021-10-22T01:34:17.948+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:34:17.947] Dropping existing documents from system.replset [2021-10-22T01:34:18.097+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:34:18.097] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:34:18.120+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:34:18.120] Inserting oplogTruncateAfterPoint document [2021-10-22T01:34:18.120+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:34:18.120] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:34:18.121+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:34:18.121] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:34:18.121+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:34:18.121] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:34:18.134+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:34:18.134] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:34:18.134+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:34:18.134] Setting the minvalid document to [{_id ObjectID("6172151a523370002becc5ca")} {t -1} {ts {0 1}}] [2021-10-22T01:34:18.147+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:34:18.147] Inserted [{_id ObjectID("6172151a523370002becc5ca")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:34:18.147+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:34:18.147] Inserting new replica set configuration document. [2021-10-22T01:34:18.147+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:34:18.147] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:34:18.159+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:34:18.159] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:34:18.164+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:34:18.164] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:34:18.164+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:34:18.164] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:34:18.164+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:34:18.164] Initiating stopEphemeralMongo [2021-10-22T01:34:18.164+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:34:18.164] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52825 (local=false) [2021-10-22T01:34:18.164+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:34:18.164] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52825 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:34:18.460+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:18.460] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:18.511+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:34:18.511] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.057547ms. [2021-10-22T01:34:18.511+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:34:18.511] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52825 (local=false) [2021-10-22T01:34:18.511+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:34:18.511] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 38.776µs. [2021-10-22T01:34:18.511+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:34:18.511] Successfully finished running stopEphemeralMongo [2021-10-22T01:34:18.511+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:34:18.511] Successfully stopped existing ephemeral mongod. [2021-10-22T01:34:18.511+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:34:18.511] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:34:18.511+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:34:18.511] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:34:18.549+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:34:18.549] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:34:18.549+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:34:18.549] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38977 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:34:18.867+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:34:18.867] clusterConfig unchanged [2021-10-22T01:34:19.462+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:19.462] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:19.729+0000] [.error] [src/action/start.go:func1:93] [103] [01:34:19.729] Error starting mongod : [01:34:19.729] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38977,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38977 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 36883 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:34:18.602+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:34:18.602+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:34:18.602+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38977,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:34:18.603+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:34:18.605+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:34:18.606+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:34:18.635+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:34:18.635+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:34:18.636+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:34:18.684+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:34:18.684+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:34:19.209+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866459:209106][36883:0x7fede67a8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:34:19.313+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866459:313582][36883:0x7fede67a8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:34:19.415+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866459:415072][36883:0x7fede67a8bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:34:19.530+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866459:530439][36883:0x7fede67a8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:34:19.607+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866459:606945][36883:0x7fede67a8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:34:19.679+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866459:679422][36883:0x7fede67a8bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:34:19.679+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866459:679494][36883:0x7fede67a8bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:34:19.681+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:34:19.729+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:34:19.729] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38977 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:34:19.729] Error starting mongod : [01:34:19.729] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38977,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38977 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 36883 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:34:18.602+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:34:18.602+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:34:18.602+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38977,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:34:18.603+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:34:18.605+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:34:18.606+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:34:18.635+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:34:18.635+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:34:18.636+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:34:18.684+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:34:18.684+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:34:19.209+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866459:209106][36883:0x7fede67a8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:34:19.313+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866459:313582][36883:0x7fede67a8bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:34:19.730+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:34:19.730] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:34:19.729] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38977 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:34:19.729] Error starting mongod : [01:34:19.729] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38977,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38977 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 36883 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:34:18.602+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:34:18.602+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:34:18.602+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38977,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:34:18.603+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:34:18.605+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:34:18.606+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:34:18.635+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:34:18.635+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:34:18.636+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:34:18.684+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:34:18.684+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:34:19.209+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866459:209106][36883:0x7fede67a8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:34:19.313+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:34:19.730+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:34:19.730] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:34:19.730+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:34:19.730] Initiating stopEphemeralMongo [2021-10-22T01:34:19.730+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:34:19.730] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:34:19.730+0000] [.error] [src/director/director.go:executePlan:962] [01:34:19.730] Failed to apply action. Result = : [01:34:19.730] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:34:19.729] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38977 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:34:19.729] Error starting mongod : [01:34:19.729] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38977,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38977 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 36883 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:34:18.602+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:34:18.602+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:34:18.602+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38977,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:34:18.603+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:34:18.605+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:34:18.606+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:34:18.635+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:34:18.635+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:34:18.636+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:34:18.684+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:34:18.684+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:34:19.209+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866459:209106][36883:0x7fede67a8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:34:19.731+0000] [.error] [src/director/director.go:planAndExecute:567] [01:34:19.730] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:34:19.730] Failed to apply action. Result = : [01:34:19.730] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:34:19.729] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38977 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:34:19.729] Error starting mongod : [01:34:19.729] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38977,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38977 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 36883 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:34:18.602+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:34:18.602+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:34:18.602+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38977,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:34:18.603+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:34:18.605+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:34:18.606+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:34:18.635+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:34:18.635+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:34:18.636+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:34:18.684+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:34:18.684+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:34:19.209+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:34:19.731+0000] [.error] [src/director/director.go:mainLoop:399] [01:34:19.731] Failed to planAndExecute : [01:34:19.730] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:34:19.730] Failed to apply action. Result = : [01:34:19.730] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:34:19.729] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38977 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:34:19.729] Error starting mongod : [01:34:19.729] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38977,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38977 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 36883 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:34:18.602+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:34:18.602+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:34:18.602+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38977,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:34:18.603+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:34:18.605+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:34:18.606+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:34:18.635+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:34:18.635+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:34:18.636+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:34:18.684+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:34:18.684+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:34:19.209+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:34:20.137+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:34:20.137] clusterConfig unchanged [2021-10-22T01:34:20.468+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:20.468] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:21.467+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:21.467] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:22.460+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:22.460] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:23.462+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:23.462] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:24.462+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:24.462] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:25.463+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:25.463] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:26.464+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:26.464] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:27.465+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:27.465] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:28.464+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:28.464] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:28.698+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:34:28.698] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.468047796s [2021-10-22T01:34:29.464+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:29.463] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:30.327+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:34:30.327] clusterConfig unchanged [2021-10-22T01:34:30.464+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:30.464] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:31.464+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:31.464] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:32.465+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:32.465] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:33.466+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:33.466] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:34.466+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:34.466] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:35.467+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:35.467] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:36.467+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:36.467] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:37.470+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:37.470] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:38.469+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:38.469] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:39.469+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:39.469] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:40.460+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:34:40.460] clusterConfig unchanged [2021-10-22T01:34:40.470+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:40.470] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:41.331+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:34:41.331] memory:rss=53993472 alloc=13693288 totalalloc=6336497368 mallocs=80815245 frees=80699675 heapinuse=17661952 heapobjects=115570 gcpausetotalns=1518747753 gcpauselastns=135908 gcnum=982 utimens=122089625000 stimens=43923754000 minflt=326288 majflt=405 [2021-10-22T01:34:41.476+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:41.476] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:42.471+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:42.471] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:43.472+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:43.472] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:44.473+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:44.473] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:45.473+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:45.473] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:46.474+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:46.474] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:47.474+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:47.474] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:48.475+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:48.475] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:49.476+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:49.476] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:50.477+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:50.477] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:50.610+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:34:50.610] clusterConfig unchanged [2021-10-22T01:34:51.477+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:51.477] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:52.477+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:52.477] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:53.478+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:53.478] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:54.478+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:54.478] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:55.478+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:55.478] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:56.478+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:56.478] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:57.478+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:57.478] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:58.479+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:58.479] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:34:59.479+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:34:59.479] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:00.480+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:00.480] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:00.743+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:35:00.743] clusterConfig unchanged [2021-10-22T01:35:01.486+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:01.486] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:02.481+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:02.481] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:03.484+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:03.484] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:04.481+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:04.481] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:05.481+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:05.481] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:06.483+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:06.483] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:07.483+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:07.483] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:08.483+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:08.483] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:09.484+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:09.484] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:10.484+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:10.484] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:10.883+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:35:10.883] clusterConfig unchanged [2021-10-22T01:35:11.485+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:11.485] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:12.492+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:12.492] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:13.486+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:13.486] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:14.487+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:14.487] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:15.487+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:15.487] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:16.488+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:16.488] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:17.488+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:17.488] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:18.489+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:18.489] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:19.506+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:19.506] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:19.905+0000] [.info] [src/director/director.go:computePlan:280] [01:35:19.905] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:35:19.906+0000] [.info] [src/director/director.go:tracef:794] [01:35:19.906] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:35:19.906+0000] [.info] [src/director/director.go:tracef:794] [01:35:19.906] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:35:19.906+0000] [.info] [src/director/director.go:tracef:794] [01:35:19.906] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:35:19.906+0000] [.info] [src/director/director.go:tracef:794] [01:35:19.906] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:35:19.906+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:35:19.906] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:35:20.071+0000] [.info] [src/director/director.go:tracef:794] [01:35:20.071] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:35:20.071+0000] [.info] [src/director/director.go:tracef:794] [01:35:20.071] because [2021-10-22T01:35:20.071+0000] [.info] [src/director/director.go:tracef:794] [01:35:20.071] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:35:20.071+0000] [.info] [src/director/director.go:tracef:794] [01:35:20.071] because ['currentState.CanBounceStop' = true] [2021-10-22T01:35:20.224+0000] [.info] [src/director/director.go:tracef:794] [01:35:20.224] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:35:20.224+0000] [.info] [src/director/director.go:tracef:794] [01:35:20.224] because [2021-10-22T01:35:20.224+0000] [.info] [src/director/director.go:tracef:794] [01:35:20.224] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:35:20.224+0000] [.info] [src/director/director.go:tracef:794] [01:35:20.224] because ['currentState.Up' = false] [2021-10-22T01:35:20.225+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:35:20.225] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:35:20.225+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:35:20.225] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:35:20.307+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:35:20.307] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:35:20.314+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:35:20.314] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:35:20.315+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:35:20.315] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:35:20.316+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:35:20.316] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:35:20.317+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:35:20.317] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:35:20.317+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:35:20.317] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:35:20.317+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:35:20.317] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:35:20.318+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:35:20.318] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:35:20.501+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:20.501] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:20.544+0000] [.info] [src/director/director.go:tracef:794] [01:35:20.544] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:35:20.545+0000] [.info] [src/director/director.go:tracef:794] [01:35:20.544] because ['currentState.Up' = false] [2021-10-22T01:35:20.757+0000] [.info] [src/director/director.go:tracef:794] [01:35:20.757] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:35:20.758+0000] [.info] [src/director/director.go:tracef:794] [01:35:20.758] because ['currentState.Up' = false] [2021-10-22T01:35:20.758+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:35:20.758] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:35:20.955+0000] [.info] [src/director/director.go:tracef:794] [01:35:20.955] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:35:20.955+0000] [.info] [src/director/director.go:tracef:794] [01:35:20.955] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:35:21.114+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:35:21.114] clusterConfig unchanged [2021-10-22T01:35:21.175+0000] [.info] [src/director/director.go:tracef:794] [01:35:21.175] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:35:21.175+0000] [.info] [src/director/director.go:tracef:794] [01:35:21.175] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:35:21.175+0000] [.info] [src/director/director.go:tracef:794] [01:35:21.175] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:35:21.175+0000] [.info] [src/director/director.go:tracef:794] [01:35:21.175] because ['currentState.Up' = false] [2021-10-22T01:35:21.175+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:35:21.175] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:35:21.175+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:35:21.175] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:35:21.211+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:35:21.211] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:35:21.211+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:35:21.211] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33837 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:35:21.503+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:21.503] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:22.008+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:35:22.008] ...Finished start [pid=56188] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33837,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:35:22.131+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:35:22.131] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 122.794527ms. [2021-10-22T01:35:22.131+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:35:22.131] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:33837 (local=false) [2021-10-22T01:35:22.131+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:35:22.131] Dropping existing documents from system.replset [2021-10-22T01:35:22.209+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:35:22.209] clusterConfig unchanged [2021-10-22T01:35:22.281+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:35:22.281] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:35:22.285+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:35:22.285] Inserting oplogTruncateAfterPoint document [2021-10-22T01:35:22.285+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:35:22.285] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:35:22.286+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:35:22.286] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:35:22.286+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:35:22.286] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:35:22.300+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:35:22.300] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:35:22.300+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:35:22.300] Setting the minvalid document to [{_id ObjectID("6172155a523370002becc5d8")} {t -1} {ts {0 1}}] [2021-10-22T01:35:22.312+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:35:22.312] Inserted [{_id ObjectID("6172155a523370002becc5d8")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:35:22.312+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:35:22.312] Inserting new replica set configuration document. [2021-10-22T01:35:22.312+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:35:22.312] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:35:22.325+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:35:22.325] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:35:22.330+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:35:22.330] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:35:22.330+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:35:22.330] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:35:22.330+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:35:22.330] Initiating stopEphemeralMongo [2021-10-22T01:35:22.330+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:35:22.330] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:33837 (local=false) [2021-10-22T01:35:22.330+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:35:22.330] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:33837 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:35:22.517+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:22.517] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:22.739+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:35:22.739] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.487586ms. [2021-10-22T01:35:22.739+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:35:22.739] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:33837 (local=false) [2021-10-22T01:35:22.739+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:35:22.739] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 29.616µs. [2021-10-22T01:35:22.739+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:35:22.739] Successfully finished running stopEphemeralMongo [2021-10-22T01:35:22.739+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:35:22.739] Successfully stopped existing ephemeral mongod. [2021-10-22T01:35:22.739+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:35:22.739] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:35:22.739+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:35:22.739] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:35:22.774+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:35:22.774] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:35:22.775+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:35:22.775] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42272 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:35:23.310+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:35:23.310] clusterConfig unchanged [2021-10-22T01:35:23.506+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:23.506] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:23.984+0000] [.error] [src/action/start.go:func1:93] [103] [01:35:23.984] Error starting mongod : [01:35:23.983] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42272,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42272 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 57761 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:35:22.828+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:35:22.828+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:35:22.828+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42272,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:35:22.830+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:35:22.831+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:35:22.832+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:35:22.864+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:35:22.864+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:35:22.865+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:35:22.916+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:35:22.916+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:35:23.467+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866523:467310][57761:0x7fb039307bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:35:23.568+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866523:568819][57761:0x7fb039307bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:35:23.671+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866523:671704][57761:0x7fb039307bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:35:23.784+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866523:784738][57761:0x7fb039307bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:35:23.867+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866523:867213][57761:0x7fb039307bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:35:23.930+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866523:930365][57761:0x7fb039307bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:35:23.930+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866523:930438][57761:0x7fb039307bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:35:23.932+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:35:23.984+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:35:23.984] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42272 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:35:23.984] Error starting mongod : [01:35:23.983] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42272,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42272 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 57761 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:35:22.828+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:35:22.828+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:35:22.828+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42272,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:35:22.830+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:35:22.831+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:35:22.832+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:35:22.864+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:35:22.864+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:35:22.865+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:35:22.916+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:35:22.916+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:35:23.467+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866523:467310][57761:0x7fb039307bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:35:23.568+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866523:568819][57761:0x7fb039307bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:35:23.984+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:35:23.984] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:35:23.984] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42272 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:35:23.984] Error starting mongod : [01:35:23.983] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42272,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42272 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 57761 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:35:22.828+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:35:22.828+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:35:22.828+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42272,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:35:22.830+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:35:22.831+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:35:22.832+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:35:22.864+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:35:22.864+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:35:22.865+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:35:22.916+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:35:22.916+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:35:23.467+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866523:467310][57761:0x7fb039307bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:35:23.568+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:35:23.984+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:35:23.984] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:35:23.984+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:35:23.984] Initiating stopEphemeralMongo [2021-10-22T01:35:23.984+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:35:23.984] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:35:23.985+0000] [.error] [src/director/director.go:executePlan:962] [01:35:23.985] Failed to apply action. Result = : [01:35:23.984] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:35:23.984] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42272 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:35:23.984] Error starting mongod : [01:35:23.983] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42272,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42272 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 57761 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:35:22.828+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:35:22.828+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:35:22.828+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42272,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:35:22.830+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:35:22.831+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:35:22.832+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:35:22.864+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:35:22.864+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:35:22.865+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:35:22.916+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:35:22.916+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:35:23.467+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866523:467310][57761:0x7fb039307bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:35:23.985+0000] [.error] [src/director/director.go:planAndExecute:567] [01:35:23.985] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:35:23.985] Failed to apply action. Result = : [01:35:23.984] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:35:23.984] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42272 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:35:23.984] Error starting mongod : [01:35:23.983] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42272,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42272 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 57761 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:35:22.828+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:35:22.828+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:35:22.828+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42272,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:35:22.830+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:35:22.831+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:35:22.832+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:35:22.864+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:35:22.864+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:35:22.865+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:35:22.916+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:35:22.916+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:35:23.467+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:35:23.985+0000] [.error] [src/director/director.go:mainLoop:399] [01:35:23.985] Failed to planAndExecute : [01:35:23.985] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:35:23.985] Failed to apply action. Result = : [01:35:23.984] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:35:23.984] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42272 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:35:23.984] Error starting mongod : [01:35:23.983] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42272,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42272 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 57761 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:35:22.828+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:35:22.828+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:35:22.828+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42272,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:35:22.830+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:35:22.831+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:35:22.832+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:35:22.864+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:35:22.864+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:35:22.865+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:35:22.916+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:35:22.916+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:35:23.467+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:35:24.506+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:24.506] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:24.580+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:35:24.580] clusterConfig unchanged [2021-10-22T01:35:25.508+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:25.508] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:26.505+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:26.505] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:27.508+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:27.508] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:28.506+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:28.506] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:29.509+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:29.509] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:30.511+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:30.511] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:31.510+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:31.510] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:32.509+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:32.509] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:32.879+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:35:32.878] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.560035964s [2021-10-22T01:35:33.509+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:33.509] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:34.509+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:34.509] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:34.701+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:35:34.701] clusterConfig unchanged [2021-10-22T01:35:35.510+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:35.510] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:36.511+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:36.511] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:37.518+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:37.518] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:38.512+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:38.512] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:39.513+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:39.513] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:40.514+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:40.514] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:41.332+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:35:41.332] memory:rss=54947840 alloc=9601888 totalalloc=6422952080 mallocs=81925356 frees=81854035 heapinuse=15884288 heapobjects=71321 gcpausetotalns=1520217786 gcpauselastns=75771 gcnum=995 utimens=124061316000 stimens=44815542000 minflt=330825 majflt=405 [2021-10-22T01:35:41.516+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:41.516] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:42.514+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:42.514] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:43.515+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:43.515] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:44.286+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [01:35:44.286] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42272 (local=false) because it is not up [2021-10-22T01:35:44.516+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:44.516] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:44.803+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:35:44.803] clusterConfig unchanged [2021-10-22T01:35:45.520+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:45.520] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:46.517+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:46.517] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:47.517+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:47.517] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:48.519+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:48.519] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:49.519+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:49.519] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:50.519+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:50.519] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:51.520+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:51.520] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:52.521+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:52.521] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:53.521+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:53.521] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:54.521+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:54.521] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:54.915+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:35:54.915] clusterConfig unchanged [2021-10-22T01:35:55.522+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:55.522] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:56.522+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:56.522] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:57.525+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:57.525] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:58.523+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:58.523] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:35:59.525+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:35:59.525] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:00.525+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:00.525] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:01.526+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:01.526] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:02.526+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:02.526] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:03.526+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:03.526] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:04.526+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:04.526] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:05.008+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:36:05.008] clusterConfig unchanged [2021-10-22T01:36:05.529+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:05.529] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:06.527+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:06.527] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:07.535+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:07.535] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:08.529+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:08.529] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:09.530+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:09.530] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:10.531+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:10.531] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:11.531+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:11.531] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:12.531+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:12.531] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:13.532+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:13.532] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:14.534+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:14.534] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:15.179+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:36:15.179] clusterConfig unchanged [2021-10-22T01:36:15.534+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:15.534] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:16.537+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:16.537] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:17.535+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:17.535] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:18.542+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:18.542] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:18.561+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:36:18.561] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:36:18.563+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:18.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:36:18.563+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [01:36:18.563] Could not get version : [01:36:18.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:36:18.563+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [01:36:18.563] Error getting version for log lines. Skipping : [01:36:18.563] Could not get version : [01:36:18.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:36:19.536+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:19.536] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:20.537+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:20.537] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:21.538+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:21.538] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:22.538+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:22.538] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:23.539+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:23.539] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:24.157+0000] [.info] [src/director/director.go:computePlan:280] [01:36:24.157] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:36:24.157+0000] [.info] [src/director/director.go:tracef:794] [01:36:24.157] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:36:24.157+0000] [.info] [src/director/director.go:tracef:794] [01:36:24.157] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:36:24.157+0000] [.info] [src/director/director.go:tracef:794] [01:36:24.157] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:36:24.157+0000] [.info] [src/director/director.go:tracef:794] [01:36:24.157] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:36:24.158+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:36:24.158] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:36:24.317+0000] [.info] [src/director/director.go:tracef:794] [01:36:24.317] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:36:24.317+0000] [.info] [src/director/director.go:tracef:794] [01:36:24.317] because [2021-10-22T01:36:24.317+0000] [.info] [src/director/director.go:tracef:794] [01:36:24.317] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:36:24.317+0000] [.info] [src/director/director.go:tracef:794] [01:36:24.317] because ['currentState.CanBounceStop' = true] [2021-10-22T01:36:24.474+0000] [.info] [src/director/director.go:tracef:794] [01:36:24.474] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:36:24.474+0000] [.info] [src/director/director.go:tracef:794] [01:36:24.474] because [2021-10-22T01:36:24.474+0000] [.info] [src/director/director.go:tracef:794] [01:36:24.474] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:36:24.474+0000] [.info] [src/director/director.go:tracef:794] [01:36:24.474] because ['currentState.Up' = false] [2021-10-22T01:36:24.474+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:36:24.474] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:36:24.474+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:36:24.474] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:36:24.540+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:24.540] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:24.553+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:36:24.553] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:36:24.562+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:36:24.562] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:36:24.563+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:36:24.563] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:36:24.563+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:36:24.563] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:36:24.564+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:36:24.564] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:36:24.564+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:36:24.564] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:36:24.564+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:36:24.564] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:36:24.566+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:36:24.566] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:36:24.770+0000] [.info] [src/director/director.go:tracef:794] [01:36:24.770] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:36:24.770+0000] [.info] [src/director/director.go:tracef:794] [01:36:24.770] because ['currentState.Up' = false] [2021-10-22T01:36:24.976+0000] [.info] [src/director/director.go:tracef:794] [01:36:24.976] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:36:24.977+0000] [.info] [src/director/director.go:tracef:794] [01:36:24.977] because ['currentState.Up' = false] [2021-10-22T01:36:24.977+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:36:24.977] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:36:25.213+0000] [.info] [src/director/director.go:tracef:794] [01:36:25.213] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:36:25.213+0000] [.info] [src/director/director.go:tracef:794] [01:36:25.213] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:36:25.402+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:36:25.402] clusterConfig unchanged [2021-10-22T01:36:25.425+0000] [.info] [src/director/director.go:tracef:794] [01:36:25.425] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:36:25.425+0000] [.info] [src/director/director.go:tracef:794] [01:36:25.425] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:36:25.425+0000] [.info] [src/director/director.go:tracef:794] [01:36:25.425] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:36:25.425+0000] [.info] [src/director/director.go:tracef:794] [01:36:25.425] because ['currentState.Up' = false] [2021-10-22T01:36:25.425+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:36:25.425] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:36:25.425+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:36:25.425] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:36:25.457+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:36:25.457] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:36:25.457+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:36:25.457] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39904 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:36:25.544+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:25.544] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:26.167+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:36:26.167] ...Finished start [pid=2946] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39904,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:36:26.308+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:36:26.308] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 140.461924ms. [2021-10-22T01:36:26.308+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:36:26.308] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39904 (local=false) [2021-10-22T01:36:26.308+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:36:26.308] Dropping existing documents from system.replset [2021-10-22T01:36:26.448+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:36:26.448] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:36:26.452+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:36:26.452] Inserting oplogTruncateAfterPoint document [2021-10-22T01:36:26.452+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:36:26.452] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:36:26.453+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:36:26.453] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:36:26.453+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:36:26.453] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:36:26.465+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:36:26.465] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:36:26.465+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:36:26.465] Setting the minvalid document to [{_id ObjectID("6172159a523370002becc5e6")} {t -1} {ts {0 1}}] [2021-10-22T01:36:26.477+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:36:26.477] Inserted [{_id ObjectID("6172159a523370002becc5e6")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:36:26.477+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:36:26.477] Inserting new replica set configuration document. [2021-10-22T01:36:26.477+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:36:26.477] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:36:26.501+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:36:26.501] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:36:26.502+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:36:26.502] clusterConfig unchanged [2021-10-22T01:36:26.505+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:36:26.505] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:36:26.505+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:36:26.505] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:36:26.505+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:36:26.505] Initiating stopEphemeralMongo [2021-10-22T01:36:26.505+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:36:26.505] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39904 (local=false) [2021-10-22T01:36:26.505+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:36:26.505] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39904 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:36:26.561+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:26.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:26.873+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:36:26.873] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.883851ms. [2021-10-22T01:36:26.873+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:36:26.873] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39904 (local=false) [2021-10-22T01:36:26.873+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:36:26.873] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.923µs. [2021-10-22T01:36:26.873+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:36:26.873] Successfully finished running stopEphemeralMongo [2021-10-22T01:36:26.873+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:36:26.873] Successfully stopped existing ephemeral mongod. [2021-10-22T01:36:26.873+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:36:26.873] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:36:26.873+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:36:26.873] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:36:26.942+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:36:26.942] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:36:26.943+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:36:26.943] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33208 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:36:27.553+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:27.553] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:27.645+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:36:27.645] clusterConfig unchanged [2021-10-22T01:36:28.206+0000] [.error] [src/action/start.go:func1:93] [103] [01:36:28.206] Error starting mongod : [01:36:28.205] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33208,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33208 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:36:27.016Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 4475 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:36:27.052+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:36:27.052+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:36:27.052+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33208,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:36:27.054+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:36:27.059+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:36:27.070+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:36:27.120+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:36:27.120+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:36:27.157+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:36:27.212+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:36:27.212+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:36:27.673+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866587:673934][4475:0x7f1f04454bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:36:27.779+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866587:779086][4475:0x7f1f04454bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:36:27.880+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866587:880092][4475:0x7f1f04454bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:36:27.993+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866587:993106][4475:0x7f1f04454bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:36:28.069+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866588:69550][4475:0x7f1f04454bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:36:28.120+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866588:120160][4475:0x7f1f04454bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:36:28.120+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866588:120228][4475:0x7f1f04454bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:36:28.206+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:36:28.206] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33208 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:36:28.206] Error starting mongod : [01:36:28.205] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33208,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33208 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:36:27.016Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 4475 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:36:27.052+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:36:27.052+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:36:27.052+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33208,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:36:27.054+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:36:27.059+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:36:27.070+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:36:27.120+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:36:27.120+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:36:27.157+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:36:27.212+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:36:27.212+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:36:27.673+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866587:673934][4475:0x7f1f04454bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:36:27.779+00:00"},"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:36:28.206+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:36:28.206] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:36:28.206] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33208 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:36:28.206] Error starting mongod : [01:36:28.205] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33208,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33208 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:36:27.016Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 4475 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:36:27.052+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:36:27.052+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:36:27.052+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33208,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:36:27.054+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:36:27.059+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:36:27.070+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:36:27.120+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:36:27.120+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:36:27.157+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:36:27.212+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:36:27.212+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:36:27.673+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866587:673934][4475:0x7f1f04454bc... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:36:28.206+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:36:28.206] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:36:28.206+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:36:28.206] Initiating stopEphemeralMongo [2021-10-22T01:36:28.206+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:36:28.206] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:36:28.207+0000] [.error] [src/director/director.go:executePlan:962] [01:36:28.207] Failed to apply action. Result = : [01:36:28.206] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:36:28.206] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33208 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:36:28.206] Error starting mongod : [01:36:28.205] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33208,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33208 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:36:27.016Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 4475 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:36:27.052+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:36:27.052+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:36:27.052+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33208,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:36:27.054+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:36:27.059+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:36:27.070+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:36:27.120+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:36:27.120+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:36:27.157+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:36:27.212+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:36:27.212+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:36:27.673+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandliste... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:36:28.207+0000] [.error] [src/director/director.go:planAndExecute:567] [01:36:28.207] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:36:28.207] Failed to apply action. Result = : [01:36:28.206] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:36:28.206] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33208 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:36:28.206] Error starting mongod : [01:36:28.205] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33208,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33208 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:36:27.016Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 4475 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:36:27.052+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:36:27.052+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:36:27.052+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33208,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:36:27.054+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:36:27.059+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:36:27.070+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:36:27.120+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:36:27.120+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:36:27.157+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:36:27.212+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:36:27.212+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"statu... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:36:28.207+0000] [.error] [src/director/director.go:mainLoop:399] [01:36:28.207] Failed to planAndExecute : [01:36:28.207] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:36:28.207] Failed to apply action. Result = : [01:36:28.206] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:36:28.206] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33208 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:36:28.206] Error starting mongod : [01:36:28.205] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33208,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33208 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:36:27.016Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 4475 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:36:27.052+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:36:27.052+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:36:27.052+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33208,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:36:27.054+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:36:27.059+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:36:27.070+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:36:27.120+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:36:27.120+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:36:27.157+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:36:27.212+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:36:27.212+00:00"},"s":"I", "c":"NETWORK", "id... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:36:28.559+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:28.559] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:28.915+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:36:28.915] clusterConfig unchanged [2021-10-22T01:36:29.558+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:29.558] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:30.559+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:30.559] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:31.559+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:31.559] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:32.560+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:32.560] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:33.561+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:33.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:34.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:34.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:35.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:35.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:36.563+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:36.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:37.563+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:37.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:37.666+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:36:37.666] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 13.100304071s [2021-10-22T01:36:38.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:38.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:39.036+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:36:39.036] clusterConfig unchanged [2021-10-22T01:36:39.561+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:39.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:40.563+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:40.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:41.332+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:36:41.332] memory:rss=53477376 alloc=10005640 totalalloc=6513735736 mallocs=83063714 frees=82988174 heapinuse=16072704 heapobjects=75540 gcpausetotalns=1522106054 gcpauselastns=111099 gcnum=1008 utimens=126079299000 stimens=45574791000 minflt=335194 majflt=405 [2021-10-22T01:36:41.563+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:41.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:42.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:42.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:43.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:43.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:44.564+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:44.564] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:45.565+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:45.565] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:46.566+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:46.566] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:47.564+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:47.564] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:48.564+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:48.564] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:49.138+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:36:49.138] clusterConfig unchanged [2021-10-22T01:36:49.564+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:49.564] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:50.565+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:50.565] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:51.566+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:51.566] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:52.567+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:52.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:53.567+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:53.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:54.568+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:54.568] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:55.569+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:55.569] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:56.570+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:56.570] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:57.576+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:57.576] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:58.570+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:58.570] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:36:59.278+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:36:59.278] clusterConfig unchanged [2021-10-22T01:36:59.571+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:36:59.571] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:00.572+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:00.572] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:01.574+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:01.574] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:02.574+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:02.574] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:03.573+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:03.573] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:04.573+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:04.573] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:05.574+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:05.574] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:06.574+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:06.574] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:07.576+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:07.576] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:08.576+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:08.576] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:09.381+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:37:09.381] clusterConfig unchanged [2021-10-22T01:37:09.577+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:09.577] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:10.577+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:10.577] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:11.578+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:11.578] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:12.581+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:12.581] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:13.579+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:13.579] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:14.579+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:14.579] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:15.580+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:15.580] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:16.581+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:16.581] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:17.581+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:17.581] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:18.581+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:18.581] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:19.522+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:37:19.522] clusterConfig unchanged [2021-10-22T01:37:19.588+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:19.588] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:20.583+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:20.582] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:21.583+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:21.583] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:22.589+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:22.589] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:23.585+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:23.585] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:24.584+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:24.583] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:25.585+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:25.585] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:26.587+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:26.587] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:27.565+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:37:27.565] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:37:27.566+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:37:27.566] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:37:27.587+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:27.587] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:28.367+0000] [.info] [src/director/director.go:computePlan:280] [01:37:28.367] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:37:28.367+0000] [.info] [src/director/director.go:tracef:794] [01:37:28.367] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:37:28.367+0000] [.info] [src/director/director.go:tracef:794] [01:37:28.367] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:37:28.368+0000] [.info] [src/director/director.go:tracef:794] [01:37:28.368] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:37:28.368+0000] [.info] [src/director/director.go:tracef:794] [01:37:28.368] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:37:28.368+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:37:28.368] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:37:28.523+0000] [.info] [src/director/director.go:tracef:794] [01:37:28.523] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:37:28.523+0000] [.info] [src/director/director.go:tracef:794] [01:37:28.523] because [2021-10-22T01:37:28.523+0000] [.info] [src/director/director.go:tracef:794] [01:37:28.523] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:37:28.523+0000] [.info] [src/director/director.go:tracef:794] [01:37:28.523] because ['currentState.CanBounceStop' = true] [2021-10-22T01:37:28.593+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:28.593] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:28.691+0000] [.info] [src/director/director.go:tracef:794] [01:37:28.691] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:37:28.691+0000] [.info] [src/director/director.go:tracef:794] [01:37:28.691] because [2021-10-22T01:37:28.691+0000] [.info] [src/director/director.go:tracef:794] [01:37:28.691] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:37:28.691+0000] [.info] [src/director/director.go:tracef:794] [01:37:28.691] because ['currentState.Up' = false] [2021-10-22T01:37:28.691+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:37:28.691] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:37:28.691+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:37:28.691] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:37:28.768+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:37:28.768] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:37:28.768+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:37:28.768] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:37:28.776+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:37:28.776] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:37:28.778+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:37:28.778] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:37:28.778+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:37:28.778] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:37:28.778+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:37:28.778] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:37:28.779+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:37:28.779] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:37:28.999+0000] [.info] [src/director/director.go:tracef:794] [01:37:28.999] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:37:28.999+0000] [.info] [src/director/director.go:tracef:794] [01:37:28.999] because ['currentState.Up' = false] [2021-10-22T01:37:29.197+0000] [.info] [src/director/director.go:tracef:794] [01:37:29.197] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:37:29.197+0000] [.info] [src/director/director.go:tracef:794] [01:37:29.197] because ['currentState.Up' = false] [2021-10-22T01:37:29.197+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:37:29.197] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:37:29.397+0000] [.info] [src/director/director.go:tracef:794] [01:37:29.397] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:37:29.398+0000] [.info] [src/director/director.go:tracef:794] [01:37:29.398] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:37:29.601+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:29.601] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:29.629+0000] [.info] [src/director/director.go:tracef:794] [01:37:29.629] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:37:29.629+0000] [.info] [src/director/director.go:tracef:794] [01:37:29.629] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:37:29.629+0000] [.info] [src/director/director.go:tracef:794] [01:37:29.629] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:37:29.629+0000] [.info] [src/director/director.go:tracef:794] [01:37:29.629] because ['currentState.Up' = false] [2021-10-22T01:37:29.629+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:37:29.629] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:37:29.629+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:37:29.629] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:37:29.662+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:37:29.662] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:37:29.663+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:37:29.662] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53650 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:37:29.713+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:37:29.713] clusterConfig unchanged [2021-10-22T01:37:30.400+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:37:30.400] ...Finished start [pid=23607] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53650,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:37:30.522+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:37:30.522] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 122.253329ms. [2021-10-22T01:37:30.522+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:37:30.522] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53650 (local=false) [2021-10-22T01:37:30.522+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:37:30.522] Dropping existing documents from system.replset [2021-10-22T01:37:30.605+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:30.605] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:30.647+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:37:30.647] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:37:30.651+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:37:30.651] Inserting oplogTruncateAfterPoint document [2021-10-22T01:37:30.651+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:37:30.651] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:37:30.652+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:37:30.652] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:37:30.652+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:37:30.652] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:37:30.665+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:37:30.665] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:37:30.665+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:37:30.665] Setting the minvalid document to [{_id ObjectID("617215da523370002becc5f4")} {t -1} {ts {0 1}}] [2021-10-22T01:37:30.678+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:37:30.678] Inserted [{_id ObjectID("617215da523370002becc5f4")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:37:30.678+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:37:30.678] Inserting new replica set configuration document. [2021-10-22T01:37:30.678+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:37:30.678] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:37:30.691+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:37:30.691] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:37:30.695+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:37:30.695] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:37:30.695+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:37:30.695] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:37:30.695+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:37:30.695] Initiating stopEphemeralMongo [2021-10-22T01:37:30.695+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:37:30.695] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53650 (local=false) [2021-10-22T01:37:30.695+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:37:30.695] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53650 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:37:30.855+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:37:30.855] clusterConfig unchanged [2021-10-22T01:37:31.184+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:37:31.184] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.150011ms. [2021-10-22T01:37:31.184+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:37:31.184] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53650 (local=false) [2021-10-22T01:37:31.184+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:37:31.184] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.405µs. [2021-10-22T01:37:31.184+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:37:31.184] Successfully finished running stopEphemeralMongo [2021-10-22T01:37:31.184+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:37:31.184] Successfully stopped existing ephemeral mongod. [2021-10-22T01:37:31.184+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:37:31.184] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:37:31.184+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:37:31.184] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:37:31.219+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:37:31.219] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:37:31.220+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:37:31.220] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35886 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:37:31.609+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:31.609] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:32.000+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:37:32.000] clusterConfig unchanged [2021-10-22T01:37:32.459+0000] [.error] [src/action/start.go:func1:93] [103] [01:37:32.459] Error starting mongod : [01:37:32.458] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35886,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35886 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:37:31.251Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 25162 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:37:31.272+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:37:31.272+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:37:31.272+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35886,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:37:31.274+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:37:31.275+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:37:31.276+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:37:31.304+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:37:31.304+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:37:31.305+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:37:31.350+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:37:31.350+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:37:31.908+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866651:908303][25162:0x7fbf20cd8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:37:32.042+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866652:42604][25162:0x7fbf20cd8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:37:32.148+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866652:148379][25162:0x7fbf20cd8bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T01:37:32.267+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866652:267308][25162:0x7fbf20cd8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:37:32.343+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866652:343412][25162:0x7fbf20cd8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:37:32.405+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866652:405823][25162:0x7fbf20cd8bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:37:32.405+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866652:405889][25162:0x7fbf20cd8bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:37:32.459+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:37:32.459] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35886 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:37:32.459] Error starting mongod : [01:37:32.458] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35886,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35886 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:37:31.251Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 25162 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:37:31.272+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:37:31.272+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:37:31.272+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35886,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:37:31.274+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:37:31.275+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:37:31.276+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:37:31.304+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:37:31.304+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:37:31.305+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:37:31.350+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:37:31.350+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:37:31.908+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866651:908303][25162:0x7fbf20cd8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:37:32.042+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:37:32.459+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:37:32.459] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:37:32.459] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35886 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:37:32.459] Error starting mongod : [01:37:32.458] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35886,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35886 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:37:31.251Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 25162 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:37:31.272+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:37:31.272+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:37:31.272+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35886,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:37:31.274+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:37:31.275+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:37:31.276+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:37:31.304+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:37:31.304+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:37:31.305+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:37:31.350+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:37:31.350+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:37:31.908+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866651:908303][25162:0x7fbf20cd8bc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:37:32.459+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:37:32.459] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:37:32.459+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:37:32.459] Initiating stopEphemeralMongo [2021-10-22T01:37:32.459+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:37:32.459] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:37:32.460+0000] [.error] [src/director/director.go:executePlan:962] [01:37:32.460] Failed to apply action. Result = : [01:37:32.459] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:37:32.459] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35886 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:37:32.459] Error starting mongod : [01:37:32.458] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35886,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35886 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:37:31.251Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 25162 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:37:31.272+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:37:31.272+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:37:31.272+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35886,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:37:31.274+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:37:31.275+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:37:31.276+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:37:31.304+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:37:31.304+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:37:31.305+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:37:31.350+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:37:31.350+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:37:31.908+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:37:32.460+0000] [.error] [src/director/director.go:planAndExecute:567] [01:37:32.460] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:37:32.460] Failed to apply action. Result = : [01:37:32.459] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:37:32.459] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35886 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:37:32.459] Error starting mongod : [01:37:32.458] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35886,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35886 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:37:31.251Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 25162 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:37:31.272+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:37:31.272+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:37:31.272+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35886,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:37:31.274+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:37:31.275+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:37:31.276+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:37:31.304+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:37:31.304+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:37:31.305+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:37:31.350+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:37:31.350+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:37:32.460+0000] [.error] [src/director/director.go:mainLoop:399] [01:37:32.460] Failed to planAndExecute : [01:37:32.460] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:37:32.460] Failed to apply action. Result = : [01:37:32.459] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:37:32.459] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35886 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:37:32.459] Error starting mongod : [01:37:32.458] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35886,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35886 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:37:31.251Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 25162 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:37:31.272+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:37:31.272+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:37:31.272+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35886,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:37:31.274+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:37:31.275+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:37:31.276+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:37:31.304+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:37:31.304+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:37:31.305+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:37:31.350+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:37:31.350+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:37:32.610+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:32.610] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:33.327+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:37:33.327] clusterConfig unchanged [2021-10-22T01:37:33.610+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:33.610] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:34.610+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:34.610] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:35.611+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:35.611] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:36.613+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [01:37:36.613] lsblk cache invalidated, running lsblk command [2021-10-22T01:37:36.616+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:36.615] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:37.611+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:37.611] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:38.612+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:38.612] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:39.614+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:39.614] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:40.613+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:40.613] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:41.333+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:37:41.333] memory:rss=54722560 alloc=14361648 totalalloc=6602072296 mallocs=84196429 frees=84078893 heapinuse=18137088 heapobjects=117536 gcpausetotalns=1523427395 gcpauselastns=127217 gcnum=1020 utimens=128125536000 stimens=46408672000 minflt=340150 majflt=405 [2021-10-22T01:37:41.474+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:37:41.474] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.695148943s [2021-10-22T01:37:41.618+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:41.618] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:42.611+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:42.611] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:43.558+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:37:43.558] clusterConfig unchanged [2021-10-22T01:37:43.612+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:43.612] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:44.612+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:44.612] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:45.613+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:45.613] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:46.615+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:46.615] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:47.614+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:47.614] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:48.614+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:48.614] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:49.615+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:49.615] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:50.616+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:50.616] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:51.621+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:51.621] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:52.618+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:52.618] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:53.618+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:53.618] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:53.699+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:37:53.699] clusterConfig unchanged [2021-10-22T01:37:54.619+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:54.619] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:55.619+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:55.619] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:56.619+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:56.619] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:57.620+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:57.620] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:58.620+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:58.620] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:37:59.621+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:37:59.621] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:00.622+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:00.622] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:01.622+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:01.622] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:02.630+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:02.630] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:03.624+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:03.624] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:03.841+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:38:03.841] clusterConfig unchanged [2021-10-22T01:38:04.625+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:04.625] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:05.626+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:05.626] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:06.626+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:06.626] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:07.627+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:07.627] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:08.627+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:08.627] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:09.627+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:09.627] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:10.628+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:10.628] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:11.629+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:11.629] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:12.629+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:12.629] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:13.630+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:13.630] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:13.972+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:38:13.972] clusterConfig unchanged [2021-10-22T01:38:14.631+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:14.631] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:15.631+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:15.631] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:16.632+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:16.632] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:17.633+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:17.633] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:18.633+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:18.633] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:19.651+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:19.651] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:20.634+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:20.634] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:21.634+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:21.634] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:22.636+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:22.636] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:23.636+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:23.636] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:24.084+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:38:24.084] clusterConfig unchanged [2021-10-22T01:38:24.642+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:24.642] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:25.636+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:25.636] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:26.636+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:26.636] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:27.638+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:27.638] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:28.639+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:28.638] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:29.639+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:29.639] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:30.639+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:30.639] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:31.639+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:31.639] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:32.627+0000] [.info] [src/director/director.go:computePlan:280] [01:38:32.627] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:38:32.627+0000] [.info] [src/director/director.go:tracef:794] [01:38:32.627] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:38:32.627+0000] [.info] [src/director/director.go:tracef:794] [01:38:32.627] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:38:32.627+0000] [.info] [src/director/director.go:tracef:794] [01:38:32.627] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:38:32.627+0000] [.info] [src/director/director.go:tracef:794] [01:38:32.627] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:38:32.627+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:38:32.627] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:38:32.647+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:32.647] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:32.785+0000] [.info] [src/director/director.go:tracef:794] [01:38:32.785] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:38:32.785+0000] [.info] [src/director/director.go:tracef:794] [01:38:32.785] because [2021-10-22T01:38:32.786+0000] [.info] [src/director/director.go:tracef:794] [01:38:32.786] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:38:32.786+0000] [.info] [src/director/director.go:tracef:794] [01:38:32.786] because ['currentState.CanBounceStop' = true] [2021-10-22T01:38:32.941+0000] [.info] [src/director/director.go:tracef:794] [01:38:32.941] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:38:32.941+0000] [.info] [src/director/director.go:tracef:794] [01:38:32.941] because [2021-10-22T01:38:32.941+0000] [.info] [src/director/director.go:tracef:794] [01:38:32.941] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:38:32.941+0000] [.info] [src/director/director.go:tracef:794] [01:38:32.941] because ['currentState.Up' = false] [2021-10-22T01:38:32.941+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:38:32.941] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:38:32.942+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:38:32.941] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:38:33.025+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:38:33.025] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:38:33.033+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:38:33.033] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:38:33.034+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:38:33.034] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:38:33.035+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:38:33.035] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:38:33.036+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:38:33.035] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:38:33.036+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:38:33.036] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:38:33.036+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:38:33.036] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:38:33.037+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:38:33.037] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:38:33.247+0000] [.info] [src/director/director.go:tracef:794] [01:38:33.247] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:38:33.247+0000] [.info] [src/director/director.go:tracef:794] [01:38:33.247] because ['currentState.Up' = false] [2021-10-22T01:38:33.443+0000] [.info] [src/director/director.go:tracef:794] [01:38:33.443] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:38:33.443+0000] [.info] [src/director/director.go:tracef:794] [01:38:33.443] because ['currentState.Up' = false] [2021-10-22T01:38:33.443+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:38:33.443] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:38:33.642+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:33.642] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:33.665+0000] [.info] [src/director/director.go:tracef:794] [01:38:33.665] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:38:33.665+0000] [.info] [src/director/director.go:tracef:794] [01:38:33.665] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:38:33.897+0000] [.info] [src/director/director.go:tracef:794] [01:38:33.897] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:38:33.897+0000] [.info] [src/director/director.go:tracef:794] [01:38:33.897] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:38:33.897+0000] [.info] [src/director/director.go:tracef:794] [01:38:33.897] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:38:33.897+0000] [.info] [src/director/director.go:tracef:794] [01:38:33.897] because ['currentState.Up' = false] [2021-10-22T01:38:33.897+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:38:33.897] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:38:33.897+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:38:33.897] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:38:33.930+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:38:33.930] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:38:33.931+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:38:33.930] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57574 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:38:34.266+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:38:34.266] clusterConfig unchanged [2021-10-22T01:38:34.643+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:34.643] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:34.661+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:38:34.661] ...Finished start [pid=44458] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57574,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:38:34.800+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:38:34.800] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 138.223558ms. [2021-10-22T01:38:34.800+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:38:34.800] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57574 (local=false) [2021-10-22T01:38:34.800+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:38:34.800] Dropping existing documents from system.replset [2021-10-22T01:38:34.934+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:38:34.933] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:38:34.938+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:38:34.938] Inserting oplogTruncateAfterPoint document [2021-10-22T01:38:34.938+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:38:34.938] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:38:34.939+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:38:34.939] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:38:34.939+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:38:34.939] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:38:34.953+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:38:34.953] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:38:34.953+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:38:34.953] Setting the minvalid document to [{_id ObjectID("6172161a523370002becc602")} {t -1} {ts {0 1}}] [2021-10-22T01:38:34.964+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:38:34.964] Inserted [{_id ObjectID("6172161a523370002becc602")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:38:34.964+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:38:34.964] Inserting new replica set configuration document. [2021-10-22T01:38:34.964+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:38:34.964] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:38:34.978+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:38:34.977] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:38:34.981+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:38:34.981] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:38:34.981+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:38:34.981] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:38:34.981+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:38:34.981] Initiating stopEphemeralMongo [2021-10-22T01:38:34.981+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:38:34.981] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57574 (local=false) [2021-10-22T01:38:34.981+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:38:34.981] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57574 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:38:35.360+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:38:35.360] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.163404ms. [2021-10-22T01:38:35.360+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:38:35.360] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57574 (local=false) [2021-10-22T01:38:35.361+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:38:35.360] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 49.772µs. [2021-10-22T01:38:35.361+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:38:35.361] Successfully finished running stopEphemeralMongo [2021-10-22T01:38:35.361+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:38:35.361] Successfully stopped existing ephemeral mongod. [2021-10-22T01:38:35.361+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:38:35.361] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:38:35.361+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:38:35.361] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:38:35.392+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:38:35.392] clusterConfig unchanged [2021-10-22T01:38:35.398+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:38:35.398] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:38:35.399+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:38:35.399] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45081 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:38:35.646+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:35.646] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:36.531+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:38:36.531] clusterConfig unchanged [2021-10-22T01:38:36.606+0000] [.error] [src/action/start.go:func1:93] [103] [01:38:36.606] Error starting mongod : [01:38:36.606] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45081,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45081 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:38:35.432Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 46030 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:38:35.463+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:38:35.463+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:38:35.463+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45081,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:38:35.465+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:38:35.466+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:38:35.467+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:38:35.500+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:38:35.500+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:38:35.500+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:38:35.546+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:38:35.546+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:38:36.091+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866716:91632][46030:0x7fa4e1fefbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:38:36.195+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866716:195729][46030:0x7fa4e1fefbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:38:36.298+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866716:298559][46030:0x7fa4e1fefbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:38:36.417+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866716:417476][46030:0x7fa4e1fefbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:38:36.496+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866716:495990][46030:0x7fa4e1fefbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:38:36.555+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866716:555510][46030:0x7fa4e1fefbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:38:36.555+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866716:555583][46030:0x7fa4e1fefbc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:38:36.606+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:38:36.606] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:45081 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:38:36.606] Error starting mongod : [01:38:36.606] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45081,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45081 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:38:35.432Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 46030 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:38:35.463+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:38:35.463+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:38:35.463+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45081,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:38:35.465+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:38:35.466+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:38:35.467+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:38:35.500+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:38:35.500+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:38:35.500+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:38:35.546+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:38:35.546+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:38:36.091+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866716:91632][46030:0x7fa4e1fefbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:38:36.195+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:38:36.607+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:38:36.607] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:38:36.606] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:45081 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:38:36.606] Error starting mongod : [01:38:36.606] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45081,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45081 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:38:35.432Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 46030 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:38:35.463+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:38:35.463+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:38:35.463+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45081,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:38:35.465+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:38:35.466+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:38:35.467+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:38:35.500+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:38:35.500+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:38:35.500+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:38:35.546+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:38:35.546+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:38:36.091+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866716:91632][46030:0x7fa4e1fefbc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:38:36.607+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:38:36.607] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:38:36.607+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:38:36.607] Initiating stopEphemeralMongo [2021-10-22T01:38:36.607+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:38:36.607] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:38:36.607+0000] [.error] [src/director/director.go:executePlan:962] [01:38:36.607] Failed to apply action. Result = : [01:38:36.607] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:38:36.606] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:45081 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:38:36.606] Error starting mongod : [01:38:36.606] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45081,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45081 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:38:35.432Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 46030 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:38:35.463+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:38:35.463+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:38:35.463+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45081,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:38:35.465+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:38:35.466+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:38:35.467+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:38:35.500+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:38:35.500+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:38:35.500+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:38:35.546+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:38:35.546+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:38:36.091+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:38:36.607+0000] [.error] [src/director/director.go:planAndExecute:567] [01:38:36.607] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:38:36.607] Failed to apply action. Result = : [01:38:36.607] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:38:36.606] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:45081 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:38:36.606] Error starting mongod : [01:38:36.606] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45081,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45081 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:38:35.432Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 46030 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:38:35.463+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:38:35.463+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:38:35.463+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45081,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:38:35.465+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:38:35.466+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:38:35.467+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:38:35.500+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:38:35.500+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:38:35.500+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:38:35.546+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:38:35.546+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:38:36.608+0000] [.error] [src/director/director.go:mainLoop:399] [01:38:36.607] Failed to planAndExecute : [01:38:36.607] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:38:36.607] Failed to apply action. Result = : [01:38:36.607] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:38:36.606] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:45081 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:38:36.606] Error starting mongod : [01:38:36.606] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45081,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45081 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:38:35.432Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 46030 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:38:35.463+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:38:35.463+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:38:35.463+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45081,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:38:35.465+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:38:35.466+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:38:35.467+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:38:35.500+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:38:35.500+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:38:35.500+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:38:35.546+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:38:35.546+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:38:36.645+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:36.645] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:37.645+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:37.645] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:37.801+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:38:37.801] clusterConfig unchanged [2021-10-22T01:38:38.646+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:38.646] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:39.647+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:39.647] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:40.648+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:40.648] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:41.335+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:38:41.335] memory:rss=55463936 alloc=10869400 totalalloc=6689540952 mallocs=85319223 frees=85236951 heapinuse=16662528 heapobjects=82272 gcpausetotalns=1524723663 gcpauselastns=110311 gcnum=1033 utimens=130020300000 stimens=47117810000 minflt=344965 majflt=405 [2021-10-22T01:38:41.648+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:41.648] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:42.649+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:42.649] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:43.649+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:43.649] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:44.297+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [01:38:44.297] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45081 (local=false) because it is not up [2021-10-22T01:38:44.650+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:44.650] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:45.509+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:38:45.509] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.47165679s [2021-10-22T01:38:45.648+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:45.648] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:46.649+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:46.649] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:47.649+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:47.649] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:47.921+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:38:47.921] clusterConfig unchanged [2021-10-22T01:38:48.650+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:48.650] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:49.651+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:49.651] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:50.652+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:50.652] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:51.653+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:51.653] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:52.653+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:52.653] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:53.653+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:53.653] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:54.655+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:54.655] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:55.655+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:55.655] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:56.655+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:56.655] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:57.656+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:57.656] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:58.043+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:38:58.043] clusterConfig unchanged [2021-10-22T01:38:58.658+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:58.658] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:38:59.657+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:38:59.657] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:00.675+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:00.675] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:01.657+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:01.657] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:02.658+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:02.658] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:03.658+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:03.658] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:04.659+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:04.659] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:05.660+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:05.659] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:06.662+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:06.662] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:07.660+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:07.660] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:08.145+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:39:08.145] clusterConfig unchanged [2021-10-22T01:39:08.662+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:08.662] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:09.662+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:09.661] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:10.662+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:10.662] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:11.663+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:11.663] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:12.663+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:12.663] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:13.663+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:13.663] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:14.664+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:14.664] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:15.664+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:15.664] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:16.665+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:16.665] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:17.672+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:17.672] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:18.267+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:39:18.267] clusterConfig unchanged [2021-10-22T01:39:18.666+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:18.666] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:19.667+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:19.667] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:20.668+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:20.668] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:21.668+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:21.668] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:22.669+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:22.669] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:23.669+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:23.669] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:24.670+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:24.670] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:25.670+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:25.670] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:26.671+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:26.671] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:27.671+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:27.671] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:28.360+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:39:28.360] clusterConfig unchanged [2021-10-22T01:39:28.678+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:28.678] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:29.672+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:29.672] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:30.672+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:30.672] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:31.673+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:31.673] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:32.674+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:32.673] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:33.674+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:33.674] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:34.675+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:34.675] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:35.675+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:35.675] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:36.677+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:36.676] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:36.773+0000] [.info] [src/director/director.go:computePlan:280] [01:39:36.773] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:39:36.773+0000] [.info] [src/director/director.go:tracef:794] [01:39:36.773] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:39:36.773+0000] [.info] [src/director/director.go:tracef:794] [01:39:36.773] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:39:36.773+0000] [.info] [src/director/director.go:tracef:794] [01:39:36.773] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:39:36.773+0000] [.info] [src/director/director.go:tracef:794] [01:39:36.773] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:39:36.774+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:39:36.774] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:39:36.933+0000] [.info] [src/director/director.go:tracef:794] [01:39:36.933] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:39:36.933+0000] [.info] [src/director/director.go:tracef:794] [01:39:36.933] because [2021-10-22T01:39:36.933+0000] [.info] [src/director/director.go:tracef:794] [01:39:36.933] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:39:36.933+0000] [.info] [src/director/director.go:tracef:794] [01:39:36.933] because ['currentState.CanBounceStop' = true] [2021-10-22T01:39:37.095+0000] [.info] [src/director/director.go:tracef:794] [01:39:37.095] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:39:37.095+0000] [.info] [src/director/director.go:tracef:794] [01:39:37.095] because [2021-10-22T01:39:37.095+0000] [.info] [src/director/director.go:tracef:794] [01:39:37.095] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:39:37.095+0000] [.info] [src/director/director.go:tracef:794] [01:39:37.095] because ['currentState.Up' = false] [2021-10-22T01:39:37.096+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:39:37.096] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:39:37.096+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:39:37.096] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:39:37.181+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:39:37.181] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:39:37.189+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:39:37.189] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:39:37.190+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:39:37.190] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:39:37.190+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:39:37.190] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:39:37.191+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:39:37.191] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:39:37.192+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:39:37.192] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:39:37.192+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:39:37.192] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:39:37.193+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:39:37.193] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:39:37.391+0000] [.info] [src/director/director.go:tracef:794] [01:39:37.391] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:39:37.391+0000] [.info] [src/director/director.go:tracef:794] [01:39:37.391] because ['currentState.Up' = false] [2021-10-22T01:39:37.616+0000] [.info] [src/director/director.go:tracef:794] [01:39:37.616] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:39:37.616+0000] [.info] [src/director/director.go:tracef:794] [01:39:37.616] because ['currentState.Up' = false] [2021-10-22T01:39:37.616+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:39:37.616] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:39:37.692+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:37.692] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:37.824+0000] [.info] [src/director/director.go:tracef:794] [01:39:37.824] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:39:37.824+0000] [.info] [src/director/director.go:tracef:794] [01:39:37.824] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:39:38.041+0000] [.info] [src/director/director.go:tracef:794] [01:39:38.041] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:39:38.041+0000] [.info] [src/director/director.go:tracef:794] [01:39:38.041] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:39:38.042+0000] [.info] [src/director/director.go:tracef:794] [01:39:38.042] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:39:38.042+0000] [.info] [src/director/director.go:tracef:794] [01:39:38.042] because ['currentState.Up' = false] [2021-10-22T01:39:38.042+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:39:38.042] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:39:38.042+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:39:38.042] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:39:38.076+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:39:38.076] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:39:38.077+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:39:38.077] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42912 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:39:38.541+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:39:38.541] clusterConfig unchanged [2021-10-22T01:39:38.680+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:38.680] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:38.789+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:39:38.789] ...Finished start [pid=65192] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42912,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:39:38.912+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:39:38.912] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 123.212814ms. [2021-10-22T01:39:38.912+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:39:38.912] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42912 (local=false) [2021-10-22T01:39:38.912+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:39:38.912] Dropping existing documents from system.replset [2021-10-22T01:39:39.051+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:39:39.051] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:39:39.054+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:39:39.054] Inserting oplogTruncateAfterPoint document [2021-10-22T01:39:39.055+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:39:39.054] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:39:39.056+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:39:39.056] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:39:39.056+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:39:39.056] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:39:39.068+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:39:39.068] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:39:39.069+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:39:39.069] Setting the minvalid document to [{_id ObjectID("6172165b523370002becc610")} {t -1} {ts {0 1}}] [2021-10-22T01:39:39.081+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:39:39.081] Inserted [{_id ObjectID("6172165b523370002becc610")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:39:39.081+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:39:39.081] Inserting new replica set configuration document. [2021-10-22T01:39:39.081+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:39:39.081] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:39:39.094+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:39:39.094] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:39:39.107+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:39:39.107] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:39:39.107+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:39:39.107] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:39:39.107+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:39:39.107] Initiating stopEphemeralMongo [2021-10-22T01:39:39.107+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:39:39.107] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42912 (local=false) [2021-10-22T01:39:39.107+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:39:39.107] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42912 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:39:39.447+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:39:39.447] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.070366ms. [2021-10-22T01:39:39.447+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:39:39.447] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42912 (local=false) [2021-10-22T01:39:39.447+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:39:39.447] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.829µs. [2021-10-22T01:39:39.447+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:39:39.447] Successfully finished running stopEphemeralMongo [2021-10-22T01:39:39.447+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:39:39.447] Successfully stopped existing ephemeral mongod. [2021-10-22T01:39:39.447+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:39:39.447] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:39:39.447+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:39:39.447] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:39:39.481+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:39:39.481] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:39:39.482+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:39:39.482] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48690 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:39:39.633+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:39:39.633] clusterConfig unchanged [2021-10-22T01:39:39.689+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:39.689] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:40.690+0000] [.error] [src/action/start.go:func1:93] [103] [01:39:40.690] Error starting mongod : [01:39:40.690] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48690,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48690 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 66636 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:39:39.532+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:39:39.532+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:39:39.532+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48690,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:39:39.533+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:39:39.535+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:39:39.536+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:39:39.694+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:39:39.694+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:39:39.696+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:39:39.930+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:39:39.930+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:39:40.159+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866780:158999][66636:0x7fbba0643bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:39:40.262+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866780:262635][66636:0x7fbba0643bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:39:40.375+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866780:375373][66636:0x7fbba0643bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:39:40.503+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866780:503112][66636:0x7fbba0643bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:39:40.581+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866780:581117][66636:0x7fbba0643bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:39:40.637+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866780:637341][66636:0x7fbba0643bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:39:40.637+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866780:637415][66636:0x7fbba0643bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:39:40.638+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:39:40.691+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:39:40.690] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48690 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:39:40.690] Error starting mongod : [01:39:40.690] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48690,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48690 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 66636 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:39:39.532+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:39:39.532+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:39:39.532+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48690,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:39:39.533+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:39:39.535+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:39:39.536+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:39:39.694+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:39:39.694+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:39:39.696+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:39:39.930+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:39:39.930+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:39:40.159+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866780:158999][66636:0x7fbba0643bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:39:40.262+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866780:262635][66636:0x7fbba0643bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:39:40.691+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:39:40.691] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:39:40.690] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48690 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:39:40.690] Error starting mongod : [01:39:40.690] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48690,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48690 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 66636 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:39:39.532+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:39:39.532+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:39:39.532+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48690,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:39:39.533+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:39:39.535+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:39:39.536+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:39:39.694+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:39:39.694+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:39:39.696+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:39:39.930+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:39:39.930+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:39:40.159+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866780:158999][66636:0x7fbba0643bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:39:40.262+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:39:40.691+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:39:40.691] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:39:40.691+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:39:40.691] Initiating stopEphemeralMongo [2021-10-22T01:39:40.691+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:39:40.691] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:39:40.692+0000] [.error] [src/director/director.go:executePlan:962] [01:39:40.691] Failed to apply action. Result = : [01:39:40.691] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:39:40.690] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48690 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:39:40.690] Error starting mongod : [01:39:40.690] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48690,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48690 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 66636 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:39:39.532+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:39:39.532+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:39:39.532+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48690,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:39:39.533+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:39:39.535+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:39:39.536+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:39:39.694+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:39:39.694+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:39:39.696+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:39:39.930+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:39:39.930+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:39:40.159+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866780:158999][66636:0x7fbba0643bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:39:40.692+0000] [.error] [src/director/director.go:planAndExecute:567] [01:39:40.692] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:39:40.691] Failed to apply action. Result = : [01:39:40.691] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:39:40.690] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48690 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:39:40.690] Error starting mongod : [01:39:40.690] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48690,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48690 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 66636 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:39:39.532+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:39:39.532+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:39:39.532+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48690,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:39:39.533+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:39:39.535+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:39:39.536+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:39:39.694+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:39:39.694+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:39:39.696+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:39:39.930+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:39:39.930+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:39:40.159+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:39:40.692+0000] [.error] [src/director/director.go:mainLoop:399] [01:39:40.692] Failed to planAndExecute : [01:39:40.692] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:39:40.691] Failed to apply action. Result = : [01:39:40.691] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:39:40.690] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48690 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:39:40.690] Error starting mongod : [01:39:40.690] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48690,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48690 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 66636 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:39:39.532+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:39:39.532+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:39:39.532+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48690,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:39:39.533+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:39:39.535+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:39:39.536+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:39:39.694+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:39:39.694+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:39:39.696+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:39:39.930+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:39:39.930+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:39:40.159+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:39:40.692+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:40.692] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:40.736+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:39:40.736] clusterConfig unchanged [2021-10-22T01:39:41.345+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:39:41.345] memory:rss=54300672 alloc=7818504 totalalloc=6777262128 mallocs=86444862 frees=86393806 heapinuse=15523840 heapobjects=51056 gcpausetotalns=1526841528 gcpauselastns=223533 gcnum=1046 utimens=131882467000 stimens=47847299000 minflt=349185 majflt=405 [2021-10-22T01:39:41.688+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:41.688] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:41.997+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:39:41.997] clusterConfig unchanged [2021-10-22T01:39:42.689+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:42.689] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:43.691+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:43.691] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:44.690+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:44.690] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:45.692+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:45.692] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:46.693+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:46.693] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:47.694+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:47.694] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:48.706+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:48.705] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:49.633+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:39:49.633] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.439524522s [2021-10-22T01:39:49.692+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:49.692] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:50.693+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:50.693] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:51.694+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:51.694] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:52.159+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:39:52.159] clusterConfig unchanged [2021-10-22T01:39:52.694+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:52.694] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:53.695+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:53.695] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:54.696+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:54.696] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:55.696+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:55.696] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:56.696+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:56.696] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:57.697+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:57.697] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:58.698+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:58.698] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:39:59.698+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:39:59.698] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:00.699+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:00.699] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:01.699+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:01.699] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:02.300+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:40:02.300] clusterConfig unchanged [2021-10-22T01:40:02.700+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:02.700] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:03.700+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:03.700] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:04.700+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:04.700] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:05.701+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:05.701] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:06.701+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:06.701] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:07.701+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:07.701] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:08.703+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:08.702] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:09.702+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:09.702] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:10.709+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:10.709] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:11.704+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:11.704] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:12.461+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:40:12.461] clusterConfig unchanged [2021-10-22T01:40:12.705+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:12.705] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:13.706+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:13.706] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:14.706+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:14.706] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:15.707+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:15.707] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:16.707+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:16.707] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:17.708+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:17.708] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:18.709+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:18.709] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:19.709+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:19.709] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:20.710+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:20.710] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:21.717+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:21.717] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:22.563+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:40:22.563] clusterConfig unchanged [2021-10-22T01:40:22.712+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:22.712] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:23.713+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:23.713] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:24.712+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:24.712] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:25.713+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:25.713] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:26.714+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:26.714] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:27.714+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:27.714] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:28.715+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:28.715] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:29.716+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:29.716] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:30.718+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:30.718] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:31.718+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:31.718] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:32.685+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:40:32.685] clusterConfig unchanged [2021-10-22T01:40:32.732+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:32.732] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:33.719+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:33.719] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:34.720+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:34.720] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:35.719+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:35.719] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:36.720+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:36.720] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:37.720+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:37.720] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:38.720+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:38.720] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:39.722+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:39.722] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:40.725+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:40.725] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:40.862+0000] [.info] [src/director/director.go:computePlan:280] [01:40:40.862] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:40:40.862+0000] [.info] [src/director/director.go:tracef:794] [01:40:40.862] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:40:40.862+0000] [.info] [src/director/director.go:tracef:794] [01:40:40.862] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:40:40.862+0000] [.info] [src/director/director.go:tracef:794] [01:40:40.862] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:40:40.862+0000] [.info] [src/director/director.go:tracef:794] [01:40:40.862] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:40:40.862+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:40:40.862] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:40:41.016+0000] [.info] [src/director/director.go:tracef:794] [01:40:41.016] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:40:41.016+0000] [.info] [src/director/director.go:tracef:794] [01:40:41.016] because [2021-10-22T01:40:41.016+0000] [.info] [src/director/director.go:tracef:794] [01:40:41.016] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:40:41.016+0000] [.info] [src/director/director.go:tracef:794] [01:40:41.016] because ['currentState.CanBounceStop' = true] [2021-10-22T01:40:41.181+0000] [.info] [src/director/director.go:tracef:794] [01:40:41.181] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:40:41.181+0000] [.info] [src/director/director.go:tracef:794] [01:40:41.181] because [2021-10-22T01:40:41.181+0000] [.info] [src/director/director.go:tracef:794] [01:40:41.181] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:40:41.181+0000] [.info] [src/director/director.go:tracef:794] [01:40:41.181] because ['currentState.Up' = false] [2021-10-22T01:40:41.181+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:40:41.181] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:40:41.181+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:40:41.181] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:40:41.261+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:40:41.261] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:40:41.269+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:40:41.269] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:40:41.270+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:40:41.270] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:40:41.270+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:40:41.270] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:40:41.271+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:40:41.271] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:40:41.271+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:40:41.271] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:40:41.271+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:40:41.271] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:40:41.273+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:40:41.273] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:40:41.345+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:40:41.345] memory:rss=54874112 alloc=12749144 totalalloc=6837037840 mallocs=87219155 frees=87105078 heapinuse=17113088 heapobjects=114077 gcpausetotalns=1527612932 gcpauselastns=116258 gcnum=1054 utimens=133316664000 stimens=48477804000 minflt=353096 majflt=405 [2021-10-22T01:40:41.482+0000] [.info] [src/director/director.go:tracef:794] [01:40:41.482] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:40:41.482+0000] [.info] [src/director/director.go:tracef:794] [01:40:41.482] because ['currentState.Up' = false] [2021-10-22T01:40:41.726+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:41.726] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:41.731+0000] [.info] [src/director/director.go:tracef:794] [01:40:41.731] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:40:41.731+0000] [.info] [src/director/director.go:tracef:794] [01:40:41.731] because ['currentState.Up' = false] [2021-10-22T01:40:41.731+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:40:41.731] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:40:41.961+0000] [.info] [src/director/director.go:tracef:794] [01:40:41.961] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:40:41.961+0000] [.info] [src/director/director.go:tracef:794] [01:40:41.961] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:40:42.173+0000] [.info] [src/director/director.go:tracef:794] [01:40:42.173] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:40:42.173+0000] [.info] [src/director/director.go:tracef:794] [01:40:42.173] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:40:42.173+0000] [.info] [src/director/director.go:tracef:794] [01:40:42.173] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:40:42.173+0000] [.info] [src/director/director.go:tracef:794] [01:40:42.173] because ['currentState.Up' = false] [2021-10-22T01:40:42.173+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:40:42.173] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:40:42.173+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:40:42.173] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:40:42.221+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:40:42.221] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:40:42.221+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:40:42.221] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48626 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:40:42.727+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:42.727] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:42.866+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:40:42.866] clusterConfig unchanged [2021-10-22T01:40:42.949+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:40:42.949] ...Finished start [pid=12023] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48626,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:40:43.104+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:40:43.104] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 155.38489ms. [2021-10-22T01:40:43.104+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:40:43.104] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:48626 (local=false) [2021-10-22T01:40:43.104+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:40:43.104] Dropping existing documents from system.replset [2021-10-22T01:40:43.290+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:40:43.290] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:40:43.297+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:40:43.297] Inserting oplogTruncateAfterPoint document [2021-10-22T01:40:43.297+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:40:43.297] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:40:43.298+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:40:43.298] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:40:43.298+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:40:43.298] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:40:43.311+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:40:43.311] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:40:43.312+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:40:43.312] Setting the minvalid document to [{_id ObjectID("6172169b523370002becc61e")} {t -1} {ts {0 1}}] [2021-10-22T01:40:43.323+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:40:43.323] Inserted [{_id ObjectID("6172169b523370002becc61e")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:40:43.323+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:40:43.323] Inserting new replica set configuration document. [2021-10-22T01:40:43.323+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:40:43.323] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:40:43.336+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:40:43.336] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:40:43.341+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:40:43.341] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:40:43.341+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:40:43.341] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:40:43.341+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:40:43.341] Initiating stopEphemeralMongo [2021-10-22T01:40:43.342+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:40:43.341] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:48626 (local=false) [2021-10-22T01:40:43.342+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:40:43.342] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:48626 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:40:43.752+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:43.752] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:43.820+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:40:43.820] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.732734ms. [2021-10-22T01:40:43.820+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:40:43.820] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:48626 (local=false) [2021-10-22T01:40:43.820+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:40:43.820] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.545µs. [2021-10-22T01:40:43.820+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:40:43.820] Successfully finished running stopEphemeralMongo [2021-10-22T01:40:43.820+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:40:43.820] Successfully stopped existing ephemeral mongod. [2021-10-22T01:40:43.821+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:40:43.820] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:40:43.821+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:40:43.821] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:40:43.859+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:40:43.859] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:40:43.860+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:40:43.859] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33861 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:40:44.007+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:40:44.007] clusterConfig unchanged [2021-10-22T01:40:44.729+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:44.729] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:45.026+0000] [.error] [src/action/start.go:func1:93] [103] [01:40:45.026] Error starting mongod : [01:40:45.025] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33861,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33861 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:40:43.892Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 13612 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:40:43.925+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:40:43.925+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:40:43.925+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33861,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:40:43.928+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:40:43.930+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:40:43.931+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:40:43.957+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:40:43.957+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:40:43.957+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:40:44.008+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:40:44.008+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:40:44.525+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866844:525323][13612:0x7f22dc8e9bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:40:44.636+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866844:636122][13612:0x7f22dc8e9bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:40:44.736+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866844:736055][13612:0x7f22dc8e9bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:40:44.849+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866844:849165][13612:0x7f22dc8e9bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:40:44.918+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866844:918148][13612:0x7f22dc8e9bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:40:44.971+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866844:971423][13612:0x7f22dc8e9bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:40:44.971+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866844:971495][13612:0x7f22dc8e9bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:40:45.026+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:40:45.026] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33861 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:40:45.026] Error starting mongod : [01:40:45.025] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33861,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33861 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:40:43.892Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 13612 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:40:43.925+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:40:43.925+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:40:43.925+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33861,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:40:43.928+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:40:43.930+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:40:43.931+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:40:43.957+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:40:43.957+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:40:43.957+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:40:44.008+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:40:44.008+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:40:44.525+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866844:525323][13612:0x7f22dc8e9bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:40:44.636+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:40:45.026+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:40:45.026] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:40:45.026] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33861 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:40:45.026] Error starting mongod : [01:40:45.025] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33861,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33861 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:40:43.892Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 13612 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:40:43.925+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:40:43.925+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:40:43.925+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33861,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:40:43.928+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:40:43.930+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:40:43.931+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:40:43.957+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:40:43.957+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:40:43.957+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:40:44.008+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:40:44.008+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:40:44.525+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866844:525323][13612:0x7f22dc8e9bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:40:45.026+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:40:45.026] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:40:45.026+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:40:45.026] Initiating stopEphemeralMongo [2021-10-22T01:40:45.026+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:40:45.026] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:40:45.027+0000] [.error] [src/director/director.go:executePlan:962] [01:40:45.027] Failed to apply action. Result = : [01:40:45.026] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:40:45.026] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33861 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:40:45.026] Error starting mongod : [01:40:45.025] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33861,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33861 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:40:43.892Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 13612 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:40:43.925+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:40:43.925+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:40:43.925+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33861,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:40:43.928+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:40:43.930+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:40:43.931+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:40:43.957+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:40:43.957+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:40:43.957+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:40:44.008+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:40:44.008+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:40:44.525+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:40:45.027+0000] [.error] [src/director/director.go:planAndExecute:567] [01:40:45.027] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:40:45.027] Failed to apply action. Result = : [01:40:45.026] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:40:45.026] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33861 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:40:45.026] Error starting mongod : [01:40:45.025] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33861,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33861 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:40:43.892Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 13612 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:40:43.925+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:40:43.925+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:40:43.925+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33861,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:40:43.928+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:40:43.930+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:40:43.931+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:40:43.957+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:40:43.957+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:40:43.957+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:40:44.008+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:40:44.008+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:40:45.027+0000] [.error] [src/director/director.go:mainLoop:399] [01:40:45.027] Failed to planAndExecute : [01:40:45.027] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:40:45.027] Failed to apply action. Result = : [01:40:45.026] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:40:45.026] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:33861 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:40:45.026] Error starting mongod : [01:40:45.025] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33861,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33861 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:40:43.892Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 13612 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:40:43.925+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:40:43.925+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:40:43.925+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33861,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:40:43.928+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:40:43.930+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:40:43.931+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:40:43.957+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:40:43.957+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:40:43.957+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:40:44.008+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:40:44.008+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:40:45.258+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:40:45.258] clusterConfig unchanged [2021-10-22T01:40:45.727+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:45.727] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:46.727+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:46.727] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:47.727+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:47.727] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:48.727+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:48.726] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:49.727+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:49.727] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:50.728+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:50.728] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:51.730+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:51.730] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:52.731+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:52.731] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:53.732+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:53.732] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:53.991+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:40:53.991] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.718035166s [2021-10-22T01:40:54.730+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:54.730] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:55.389+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:40:55.389] clusterConfig unchanged [2021-10-22T01:40:55.730+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:55.730] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:56.731+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:56.731] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:57.731+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:57.731] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:58.734+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:58.734] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:40:59.732+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:40:59.732] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:00.733+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:00.733] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:01.734+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:01.734] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:02.735+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:02.735] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:03.736+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:03.736] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:04.736+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:04.736] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:05.511+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:41:05.511] clusterConfig unchanged [2021-10-22T01:41:05.737+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:05.737] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:06.738+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:06.738] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:07.738+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:07.738] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:08.739+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:08.739] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:09.739+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:09.739] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:10.747+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:10.747] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:11.741+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:11.741] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:12.742+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:12.742] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:13.742+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:13.742] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:14.743+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:14.743] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:15.692+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:41:15.692] clusterConfig unchanged [2021-10-22T01:41:15.743+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:15.743] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:16.744+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:16.744] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:17.745+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:17.745] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:18.562+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:41:18.562] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:41:18.564+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:18.564] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:41:18.564+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [01:41:18.564] Could not get version : [01:41:18.564] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:41:18.564+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [01:41:18.564] Error getting version for log lines. Skipping : [01:41:18.564] Could not get version : [01:41:18.564] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:41:18.745+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:18.745] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:19.751+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:19.751] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:20.747+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:20.747] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:21.747+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:21.747] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:22.748+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:22.748] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:23.748+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:23.748] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:24.749+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:24.749] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:25.749+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:25.749] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:25.824+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:41:25.824] clusterConfig unchanged [2021-10-22T01:41:26.750+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:26.750] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:27.750+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:27.750] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:28.751+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:28.751] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:29.751+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:29.751] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:30.751+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:30.751] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:31.759+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:31.759] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:32.753+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:32.753] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:33.753+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:33.753] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:34.754+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:34.754] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:35.754+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:35.754] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:35.926+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:41:35.926] clusterConfig unchanged [2021-10-22T01:41:36.755+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:36.755] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:37.756+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:37.756] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:38.757+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:38.757] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:39.759+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:39.759] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:40.757+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:40.757] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:41.347+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:41:41.347] memory:rss=54185984 alloc=13037368 totalalloc=6904070464 mallocs=88030248 frees=87920088 heapinuse=16785408 heapobjects=110160 gcpausetotalns=1529886090 gcpauselastns=110035 gcnum=1063 utimens=135269882000 stimens=49286908000 minflt=357527 majflt=405 [2021-10-22T01:41:41.758+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:41.758] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:42.758+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:42.758] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:43.758+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:43.758] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:44.306+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [01:41:44.306] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:33861 (local=false) because it is not up [2021-10-22T01:41:44.759+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:44.759] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:45.195+0000] [.info] [src/director/director.go:computePlan:280] [01:41:45.195] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:41:45.195+0000] [.info] [src/director/director.go:tracef:794] [01:41:45.195] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:41:45.195+0000] [.info] [src/director/director.go:tracef:794] [01:41:45.195] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:41:45.195+0000] [.info] [src/director/director.go:tracef:794] [01:41:45.195] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:41:45.195+0000] [.info] [src/director/director.go:tracef:794] [01:41:45.195] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:41:45.196+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:41:45.196] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:41:45.349+0000] [.info] [src/director/director.go:tracef:794] [01:41:45.349] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:41:45.349+0000] [.info] [src/director/director.go:tracef:794] [01:41:45.349] because [2021-10-22T01:41:45.349+0000] [.info] [src/director/director.go:tracef:794] [01:41:45.349] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:41:45.349+0000] [.info] [src/director/director.go:tracef:794] [01:41:45.349] because ['currentState.CanBounceStop' = true] [2021-10-22T01:41:45.498+0000] [.info] [src/director/director.go:tracef:794] [01:41:45.498] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:41:45.498+0000] [.info] [src/director/director.go:tracef:794] [01:41:45.498] because [2021-10-22T01:41:45.499+0000] [.info] [src/director/director.go:tracef:794] [01:41:45.499] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:41:45.499+0000] [.info] [src/director/director.go:tracef:794] [01:41:45.499] because ['currentState.Up' = false] [2021-10-22T01:41:45.499+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:41:45.499] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:41:45.499+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:41:45.499] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:41:45.585+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:41:45.585] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:41:45.593+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:41:45.593] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:41:45.594+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:41:45.594] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:41:45.595+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:41:45.595] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:41:45.596+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:41:45.596] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:41:45.596+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:41:45.596] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:41:45.596+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:41:45.596] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:41:45.597+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:41:45.597] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:41:45.762+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:45.762] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:45.820+0000] [.info] [src/director/director.go:tracef:794] [01:41:45.820] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:41:45.820+0000] [.info] [src/director/director.go:tracef:794] [01:41:45.820] because ['currentState.Up' = false] [2021-10-22T01:41:46.024+0000] [.info] [src/director/director.go:tracef:794] [01:41:46.024] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:41:46.024+0000] [.info] [src/director/director.go:tracef:794] [01:41:46.024] because ['currentState.Up' = false] [2021-10-22T01:41:46.024+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:41:46.024] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:41:46.141+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:41:46.141] clusterConfig unchanged [2021-10-22T01:41:46.224+0000] [.info] [src/director/director.go:tracef:794] [01:41:46.224] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:41:46.224+0000] [.info] [src/director/director.go:tracef:794] [01:41:46.224] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:41:46.450+0000] [.info] [src/director/director.go:tracef:794] [01:41:46.450] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:41:46.450+0000] [.info] [src/director/director.go:tracef:794] [01:41:46.450] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:41:46.451+0000] [.info] [src/director/director.go:tracef:794] [01:41:46.451] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:41:46.451+0000] [.info] [src/director/director.go:tracef:794] [01:41:46.451] because ['currentState.Up' = false] [2021-10-22T01:41:46.451+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:41:46.451] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:41:46.451+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:41:46.451] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:41:46.501+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:41:46.501] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:41:46.502+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:41:46.502] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39459 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:41:46.795+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:46.795] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:47.244+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:41:47.244] ...Finished start [pid=32649] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39459,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:41:47.259+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:41:47.259] clusterConfig unchanged [2021-10-22T01:41:47.368+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:41:47.368] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 123.498427ms. [2021-10-22T01:41:47.368+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:41:47.368] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39459 (local=false) [2021-10-22T01:41:47.368+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:41:47.368] Dropping existing documents from system.replset [2021-10-22T01:41:47.508+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:41:47.508] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:41:47.521+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:41:47.521] Inserting oplogTruncateAfterPoint document [2021-10-22T01:41:47.521+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:41:47.521] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:41:47.522+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:41:47.522] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:41:47.522+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:41:47.522] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:41:47.537+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:41:47.537] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:41:47.537+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:41:47.537] Setting the minvalid document to [{_id ObjectID("617216db523370002becc62c")} {t -1} {ts {0 1}}] [2021-10-22T01:41:47.550+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:41:47.550] Inserted [{_id ObjectID("617216db523370002becc62c")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:41:47.550+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:41:47.550] Inserting new replica set configuration document. [2021-10-22T01:41:47.550+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:41:47.550] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:41:47.563+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:41:47.563] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:41:47.566+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:41:47.566] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:41:47.566+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:41:47.566] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:41:47.566+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:41:47.566] Initiating stopEphemeralMongo [2021-10-22T01:41:47.566+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:41:47.566] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39459 (local=false) [2021-10-22T01:41:47.567+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:41:47.566] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39459 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:41:47.770+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:47.770] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:47.971+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:41:47.971] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.112707ms. [2021-10-22T01:41:47.971+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:41:47.971] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39459 (local=false) [2021-10-22T01:41:47.971+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:41:47.971] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 40.464µs. [2021-10-22T01:41:47.971+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:41:47.971] Successfully finished running stopEphemeralMongo [2021-10-22T01:41:47.972+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:41:47.971] Successfully stopped existing ephemeral mongod. [2021-10-22T01:41:47.972+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:41:47.972] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:41:47.972+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:41:47.972] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:41:48.008+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:41:48.008] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:41:48.009+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:41:48.009] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52783 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:41:48.392+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:41:48.392] clusterConfig unchanged [2021-10-22T01:41:48.779+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:48.779] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:49.280+0000] [.error] [src/action/start.go:func1:93] [103] [01:41:49.280] Error starting mongod : [01:41:49.280] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52783,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52783 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 34225 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:41:48.068+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:41:48.068+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:41:48.068+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52783,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:41:48.069+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:41:48.071+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:41:48.093+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:41:48.178+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:41:48.178+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:41:48.178+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:41:48.229+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:41:48.229+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:41:48.730+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866908:730770][34225:0x7f4ec5f15bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:41:48.833+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866908:833664][34225:0x7f4ec5f15bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:41:48.932+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866908:932833][34225:0x7f4ec5f15bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:41:49.071+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866909:71308][34225:0x7f4ec5f15bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:41:49.160+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866909:160915][34225:0x7f4ec5f15bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:41:49.218+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866909:218671][34225:0x7f4ec5f15bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:41:49.218+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866909:218745][34225:0x7f4ec5f15bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:41:49.220+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:41:49.280+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:41:49.280] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:52783 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:41:49.280] Error starting mongod : [01:41:49.280] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52783,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52783 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 34225 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:41:48.068+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:41:48.068+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:41:48.068+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52783,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:41:48.069+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:41:48.071+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:41:48.093+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:41:48.178+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:41:48.178+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:41:48.178+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:41:48.229+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:41:48.229+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:41:48.730+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866908:730770][34225:0x7f4ec5f15bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:41:48.833+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866908:833664][34225:0x7f4ec5f15bc0], txn-recover: [W...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:41:49.280+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:41:49.280] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:41:49.280] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:52783 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:41:49.280] Error starting mongod : [01:41:49.280] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52783,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52783 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 34225 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:41:48.068+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:41:48.068+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:41:48.068+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52783,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:41:48.069+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:41:48.071+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:41:48.093+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:41:48.178+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:41:48.178+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:41:48.178+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:41:48.229+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:41:48.229+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:41:48.730+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866908:730770][34225:0x7f4ec5f15bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:41:48.833+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ct... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:41:49.281+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:41:49.281] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:41:49.281+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:41:49.281] Initiating stopEphemeralMongo [2021-10-22T01:41:49.281+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:41:49.281] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:41:49.281+0000] [.error] [src/director/director.go:executePlan:962] [01:41:49.281] Failed to apply action. Result = : [01:41:49.280] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:41:49.280] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:52783 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:41:49.280] Error starting mongod : [01:41:49.280] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52783,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52783 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 34225 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:41:48.068+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:41:48.068+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:41:48.068+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52783,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:41:48.069+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:41:48.071+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:41:48.093+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:41:48.178+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:41:48.178+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:41:48.178+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:41:48.229+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:41:48.229+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:41:48.730+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866908:730770][34225:0x7f4ec5f15bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:41:49.281+0000] [.error] [src/director/director.go:planAndExecute:567] [01:41:49.281] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:41:49.281] Failed to apply action. Result = : [01:41:49.280] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:41:49.280] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:52783 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:41:49.280] Error starting mongod : [01:41:49.280] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52783,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52783 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 34225 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:41:48.068+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:41:48.068+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:41:48.068+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52783,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:41:48.069+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:41:48.071+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:41:48.093+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:41:48.178+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:41:48.178+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:41:48.178+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:41:48.229+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:41:48.229+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:41:48.730+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:41:49.281+0000] [.error] [src/director/director.go:mainLoop:399] [01:41:49.281] Failed to planAndExecute : [01:41:49.281] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:41:49.281] Failed to apply action. Result = : [01:41:49.280] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:41:49.280] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:52783 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:41:49.280] Error starting mongod : [01:41:49.280] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52783,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52783 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 34225 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:41:48.068+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:41:48.068+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:41:48.068+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52783,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:41:48.069+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:41:48.071+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:41:48.093+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:41:48.178+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:41:48.178+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:41:48.178+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:41:48.229+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:41:48.229+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:41:48.730+00:00"},"s":"I", "c":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:41:49.622+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:41:49.622] clusterConfig unchanged [2021-10-22T01:41:49.779+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:49.779] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:50.780+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:50.780] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:51.778+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:51.778] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:52.780+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:52.780] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:53.781+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:53.781] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:54.796+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:54.796] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:55.780+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:55.780] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:56.784+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:56.784] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:57.783+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:57.783] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:58.157+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:41:58.156] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.559280346s [2021-10-22T01:41:58.781+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:58.781] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:41:59.744+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:41:59.744] clusterConfig unchanged [2021-10-22T01:41:59.782+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:41:59.782] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:00.783+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:00.783] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:01.785+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:01.785] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:02.786+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:02.786] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:03.788+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:03.788] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:04.785+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:04.785] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:05.792+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:05.792] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:06.788+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:06.788] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:07.787+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:07.787] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:08.787+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:08.787] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:09.788+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:09.788] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:09.885+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:42:09.885] clusterConfig unchanged [2021-10-22T01:42:10.789+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:10.789] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:11.790+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:11.789] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:12.790+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:12.790] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:13.791+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:13.791] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:14.791+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:14.791] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:15.792+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:15.792] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:16.795+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:16.795] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:17.792+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:17.792] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:18.793+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:18.793] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:19.794+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:19.794] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:20.007+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:42:20.007] clusterConfig unchanged [2021-10-22T01:42:20.794+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:20.794] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:21.794+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:21.794] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:22.795+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:22.795] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:23.796+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:23.796] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:24.797+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:24.797] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:25.797+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:25.797] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:26.799+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:26.799] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:27.566+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:42:27.566] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:42:27.567+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:42:27.567] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:42:27.804+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:27.804] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:28.799+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:28.799] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:29.800+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:29.800] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:30.119+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:42:30.119] clusterConfig unchanged [2021-10-22T01:42:30.800+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:30.800] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:31.801+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:31.801] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:32.801+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:32.801] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:33.801+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:33.801] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:34.802+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:34.802] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:35.803+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:35.803] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:36.806+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [01:42:36.806] lsblk cache invalidated, running lsblk command [2021-10-22T01:42:36.807+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:36.807] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:37.807+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:37.807] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:38.805+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:38.805] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:39.814+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:39.814] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:40.231+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:42:40.231] clusterConfig unchanged [2021-10-22T01:42:40.805+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:40.805] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:41.347+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:42:41.347] memory:rss=54239232 alloc=8612256 totalalloc=6993791744 mallocs=89166885 frees=89105149 heapinuse=15360000 heapobjects=61736 gcpausetotalns=1531127197 gcpauselastns=78630 gcnum=1076 utimens=137354211000 stimens=50132941000 minflt=362233 majflt=405 [2021-10-22T01:42:41.806+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:41.806] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:42.806+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:42.806] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:43.807+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:43.807] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:44.807+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:44.807] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:45.808+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:45.808] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:46.808+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:46.808] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:47.809+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:47.809] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:48.809+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:48.809] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:49.449+0000] [.info] [src/director/director.go:computePlan:280] [01:42:49.449] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:42:49.449+0000] [.info] [src/director/director.go:tracef:794] [01:42:49.449] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:42:49.449+0000] [.info] [src/director/director.go:tracef:794] [01:42:49.449] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:42:49.449+0000] [.info] [src/director/director.go:tracef:794] [01:42:49.449] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:42:49.449+0000] [.info] [src/director/director.go:tracef:794] [01:42:49.449] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:42:49.449+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:42:49.449] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:42:49.600+0000] [.info] [src/director/director.go:tracef:794] [01:42:49.600] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:42:49.600+0000] [.info] [src/director/director.go:tracef:794] [01:42:49.600] because [2021-10-22T01:42:49.600+0000] [.info] [src/director/director.go:tracef:794] [01:42:49.600] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:42:49.600+0000] [.info] [src/director/director.go:tracef:794] [01:42:49.600] because ['currentState.CanBounceStop' = true] [2021-10-22T01:42:49.757+0000] [.info] [src/director/director.go:tracef:794] [01:42:49.757] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:42:49.757+0000] [.info] [src/director/director.go:tracef:794] [01:42:49.757] because [2021-10-22T01:42:49.757+0000] [.info] [src/director/director.go:tracef:794] [01:42:49.757] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:42:49.757+0000] [.info] [src/director/director.go:tracef:794] [01:42:49.757] because ['currentState.Up' = false] [2021-10-22T01:42:49.757+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:42:49.757] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:42:49.757+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:42:49.757] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:42:49.810+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:49.810] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:49.841+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:42:49.841] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:42:49.849+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:42:49.849] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:42:49.850+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:42:49.850] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:42:49.851+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:42:49.851] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:42:49.851+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:42:49.851] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:42:49.851+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:42:49.851] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:42:49.852+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:42:49.852] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:42:50.157+0000] [.info] [src/director/director.go:tracef:794] [01:42:50.157] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:42:50.157+0000] [.info] [src/director/director.go:tracef:794] [01:42:50.157] because ['currentState.Up' = false] [2021-10-22T01:42:50.358+0000] [.info] [src/director/director.go:tracef:794] [01:42:50.358] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:42:50.358+0000] [.info] [src/director/director.go:tracef:794] [01:42:50.358] because ['currentState.Up' = false] [2021-10-22T01:42:50.358+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:42:50.358] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:42:50.411+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:42:50.411] clusterConfig unchanged [2021-10-22T01:42:50.578+0000] [.info] [src/director/director.go:tracef:794] [01:42:50.578] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:42:50.578+0000] [.info] [src/director/director.go:tracef:794] [01:42:50.578] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:42:50.788+0000] [.info] [src/director/director.go:tracef:794] [01:42:50.788] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:42:50.788+0000] [.info] [src/director/director.go:tracef:794] [01:42:50.788] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:42:50.789+0000] [.info] [src/director/director.go:tracef:794] [01:42:50.789] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:42:50.789+0000] [.info] [src/director/director.go:tracef:794] [01:42:50.789] because ['currentState.Up' = false] [2021-10-22T01:42:50.789+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:42:50.789] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:42:50.789+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:42:50.789] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:42:50.813+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:50.813] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:50.822+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:42:50.822] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:42:50.823+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:42:50.823] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50453 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:42:51.504+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:42:51.504] clusterConfig unchanged [2021-10-22T01:42:51.547+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:42:51.547] ...Finished start [pid=53428] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50453,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:42:51.689+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:42:51.689] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 141.279397ms. [2021-10-22T01:42:51.689+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:42:51.689] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50453 (local=false) [2021-10-22T01:42:51.689+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:42:51.689] Dropping existing documents from system.replset [2021-10-22T01:42:51.819+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:51.819] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:51.839+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:42:51.839] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:42:51.844+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:42:51.844] Inserting oplogTruncateAfterPoint document [2021-10-22T01:42:51.844+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:42:51.844] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:42:51.845+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:42:51.845] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:42:51.845+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:42:51.845] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:42:51.857+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:42:51.857] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:42:51.857+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:42:51.857] Setting the minvalid document to [{_id ObjectID("6172171b523370002becc63a")} {t -1} {ts {0 1}}] [2021-10-22T01:42:51.870+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:42:51.870] Inserted [{_id ObjectID("6172171b523370002becc63a")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:42:51.870+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:42:51.870] Inserting new replica set configuration document. [2021-10-22T01:42:51.870+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:42:51.870] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:42:51.884+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:42:51.884] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:42:51.887+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:42:51.887] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:42:51.887+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:42:51.887] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:42:51.887+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:42:51.887] Initiating stopEphemeralMongo [2021-10-22T01:42:51.887+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:42:51.887] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50453 (local=false) [2021-10-22T01:42:51.887+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:42:51.887] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50453 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:42:52.324+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:42:52.324] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.131929ms. [2021-10-22T01:42:52.324+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:42:52.324] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50453 (local=false) [2021-10-22T01:42:52.324+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:42:52.324] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 52.665µs. [2021-10-22T01:42:52.324+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:42:52.324] Successfully finished running stopEphemeralMongo [2021-10-22T01:42:52.324+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:42:52.324] Successfully stopped existing ephemeral mongod. [2021-10-22T01:42:52.324+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:42:52.324] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:42:52.324+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:42:52.324] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:42:52.361+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:42:52.361] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:42:52.362+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:42:52.362] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37363 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:42:52.618+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:42:52.618] clusterConfig unchanged [2021-10-22T01:42:52.820+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:52.820] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:53.522+0000] [.error] [src/action/start.go:func1:93] [103] [01:42:53.522] Error starting mongod : [01:42:53.522] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37363,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37363 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 54974 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:42:52.414+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:42:52.414+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:42:52.414+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37363,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:42:52.416+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:42:52.417+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:42:52.418+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:42:52.448+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:42:52.448+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:42:52.473+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:42:52.526+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:42:52.526+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:42:53.014+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866973:14125][54974:0x7fe51c021bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:42:53.121+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866973:120986][54974:0x7fe51c021bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:42:53.223+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866973:223350][54974:0x7fe51c021bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T01:42:53.336+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866973:336029][54974:0x7fe51c021bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:42:53.413+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866973:413648][54974:0x7fe51c021bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:42:53.474+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866973:474678][54974:0x7fe51c021bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:42:53.474+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866973:474742][54974:0x7fe51c021bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:42:53.476+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:42:53.522+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:42:53.522] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37363 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:42:53.522] Error starting mongod : [01:42:53.522] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37363,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37363 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 54974 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:42:52.414+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:42:52.414+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:42:52.414+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37363,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:42:52.416+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:42:52.417+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:42:52.418+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:42:52.448+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:42:52.448+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:42:52.473+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:42:52.526+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:42:52.526+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:42:53.014+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866973:14125][54974:0x7fe51c021bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:42:53.121+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866973:120986][54974:0x7fe51c021bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:42:53.523+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:42:53.522] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:42:53.522] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37363 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:42:53.522] Error starting mongod : [01:42:53.522] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37363,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37363 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 54974 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:42:52.414+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:42:52.414+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:42:52.414+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37363,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:42:52.416+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:42:52.417+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:42:52.418+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:42:52.448+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:42:52.448+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:42:52.473+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:42:52.526+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:42:52.526+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:42:53.014+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866973:14125][54974:0x7fe51c021bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:42:53.121+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:42:53.523+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:42:53.523] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:42:53.523+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:42:53.523] Initiating stopEphemeralMongo [2021-10-22T01:42:53.523+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:42:53.523] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:42:53.523+0000] [.error] [src/director/director.go:executePlan:962] [01:42:53.523] Failed to apply action. Result = : [01:42:53.522] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:42:53.522] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37363 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:42:53.522] Error starting mongod : [01:42:53.522] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37363,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37363 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 54974 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:42:52.414+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:42:52.414+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:42:52.414+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37363,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:42:52.416+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:42:52.417+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:42:52.418+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:42:52.448+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:42:52.448+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:42:52.473+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:42:52.526+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:42:52.526+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:42:53.014+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634866973:14125][54974:0x7fe51c021bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:42:53.523+0000] [.error] [src/director/director.go:planAndExecute:567] [01:42:53.523] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:42:53.523] Failed to apply action. Result = : [01:42:53.522] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:42:53.522] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37363 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:42:53.522] Error starting mongod : [01:42:53.522] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37363,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37363 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 54974 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:42:52.414+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:42:52.414+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:42:52.414+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37363,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:42:52.416+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:42:52.417+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:42:52.418+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:42:52.448+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:42:52.448+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:42:52.473+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:42:52.526+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:42:52.526+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:42:53.014+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:42:53.523+0000] [.error] [src/director/director.go:mainLoop:399] [01:42:53.523] Failed to planAndExecute : [01:42:53.523] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:42:53.523] Failed to apply action. Result = : [01:42:53.522] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:42:53.522] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37363 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:42:53.522] Error starting mongod : [01:42:53.522] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37363,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37363 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 54974 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:42:52.414+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:42:52.414+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:42:52.414+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37363,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:42:52.416+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:42:52.417+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:42:52.418+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:42:52.448+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:42:52.448+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:42:52.473+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:42:52.526+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:42:52.526+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:42:53.014+00:00"},"s":"I", "c":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:42:53.820+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:53.820] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:53.858+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:42:53.858] clusterConfig unchanged [2021-10-22T01:42:54.822+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:54.822] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:55.838+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:55.838] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:56.822+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:56.822] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:57.829+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:57.829] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:58.825+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:58.825] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:42:59.824+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:42:59.824] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:00.826+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:00.826] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:01.827+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:01.827] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:02.516+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:43:02.516] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.66328534s [2021-10-22T01:43:02.828+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:02.828] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:03.825+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:03.825] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:03.962+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:43:03.962] clusterConfig unchanged [2021-10-22T01:43:04.825+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:04.825] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:05.826+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:05.826] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:06.826+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:06.826] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:07.827+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:07.827] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:08.828+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:08.828] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:09.828+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:09.828] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:10.829+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:10.829] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:11.829+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:11.829] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:12.829+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:12.829] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:13.830+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:13.830] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:14.052+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:43:14.052] clusterConfig unchanged [2021-10-22T01:43:14.836+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:14.836] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:15.830+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:15.830] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:16.831+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:16.831] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:17.831+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:17.831] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:18.833+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:18.833] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:19.833+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:19.833] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:20.834+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:20.834] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:21.834+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:21.834] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:22.835+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:22.835] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:23.836+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:23.836] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:24.184+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:43:24.184] clusterConfig unchanged [2021-10-22T01:43:24.837+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:24.837] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:25.837+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:25.837] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:26.845+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:26.845] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:27.837+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:27.837] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:28.838+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:28.838] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:29.839+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:29.839] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:30.840+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:30.840] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:31.840+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:31.840] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:32.841+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:32.841] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:33.842+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:33.842] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:34.276+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:43:34.276] clusterConfig unchanged [2021-10-22T01:43:34.843+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:34.843] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:35.843+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:35.843] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:36.843+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:36.843] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:37.844+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:37.844] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:38.845+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:38.845] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:39.847+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:39.847] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:40.847+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:40.847] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:41.348+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:43:41.348] memory:rss=53309440 alloc=9273032 totalalloc=7082767960 mallocs=90304153 frees=90235266 heapinuse=15564800 heapobjects=68887 gcpausetotalns=1532519942 gcpauselastns=98124 gcnum=1088 utimens=139419200000 stimens=51020189000 minflt=367581 majflt=405 [2021-10-22T01:43:41.847+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:41.847] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:42.848+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:42.848] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:43.849+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:43.849] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:44.408+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:43:44.408] clusterConfig unchanged [2021-10-22T01:43:44.849+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:44.849] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:45.849+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:45.849] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:46.856+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:46.856] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:47.850+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:47.850] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:48.851+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:48.851] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:49.851+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:49.851] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:50.852+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:50.852] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:51.853+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:51.853] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:52.853+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:52.853] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:53.688+0000] [.info] [src/director/director.go:computePlan:280] [01:43:53.688] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:43:53.688+0000] [.info] [src/director/director.go:tracef:794] [01:43:53.688] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:43:53.688+0000] [.info] [src/director/director.go:tracef:794] [01:43:53.688] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:43:53.688+0000] [.info] [src/director/director.go:tracef:794] [01:43:53.688] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:43:53.688+0000] [.info] [src/director/director.go:tracef:794] [01:43:53.688] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:43:53.688+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:43:53.688] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:43:53.842+0000] [.info] [src/director/director.go:tracef:794] [01:43:53.842] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:43:53.842+0000] [.info] [src/director/director.go:tracef:794] [01:43:53.842] because [2021-10-22T01:43:53.842+0000] [.info] [src/director/director.go:tracef:794] [01:43:53.842] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:43:53.842+0000] [.info] [src/director/director.go:tracef:794] [01:43:53.842] because ['currentState.CanBounceStop' = true] [2021-10-22T01:43:53.856+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:53.856] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:53.998+0000] [.info] [src/director/director.go:tracef:794] [01:43:53.998] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:43:53.998+0000] [.info] [src/director/director.go:tracef:794] [01:43:53.998] because [2021-10-22T01:43:53.998+0000] [.info] [src/director/director.go:tracef:794] [01:43:53.998] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:43:53.998+0000] [.info] [src/director/director.go:tracef:794] [01:43:53.998] because ['currentState.Up' = false] [2021-10-22T01:43:53.998+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:43:53.998] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:43:53.998+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:43:53.998] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:43:54.077+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:43:54.077] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:43:54.078+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:43:54.078] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:43:54.086+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:43:54.086] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:43:54.088+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:43:54.088] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:43:54.090+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:43:54.090] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:43:54.090+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:43:54.090] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:43:54.092+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:43:54.092] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:43:54.094+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:43:54.094] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:43:54.315+0000] [.info] [src/director/director.go:tracef:794] [01:43:54.315] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:43:54.315+0000] [.info] [src/director/director.go:tracef:794] [01:43:54.315] because ['currentState.Up' = false] [2021-10-22T01:43:54.523+0000] [.info] [src/director/director.go:tracef:794] [01:43:54.523] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:43:54.523+0000] [.info] [src/director/director.go:tracef:794] [01:43:54.523] because ['currentState.Up' = false] [2021-10-22T01:43:54.523+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:43:54.523] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:43:54.609+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:43:54.609] clusterConfig unchanged [2021-10-22T01:43:54.732+0000] [.info] [src/director/director.go:tracef:794] [01:43:54.732] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:43:54.732+0000] [.info] [src/director/director.go:tracef:794] [01:43:54.732] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:43:54.857+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:54.857] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:54.925+0000] [.info] [src/director/director.go:tracef:794] [01:43:54.925] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:43:54.925+0000] [.info] [src/director/director.go:tracef:794] [01:43:54.925] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:43:54.926+0000] [.info] [src/director/director.go:tracef:794] [01:43:54.926] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:43:54.926+0000] [.info] [src/director/director.go:tracef:794] [01:43:54.926] because ['currentState.Up' = false] [2021-10-22T01:43:54.926+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:43:54.926] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:43:54.926+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:43:54.926] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:43:54.958+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:43:54.958] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:43:54.958+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:43:54.958] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38307 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:43:55.687+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:43:55.687] ...Finished start [pid=74149] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38307,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:43:55.702+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:43:55.702] clusterConfig unchanged [2021-10-22T01:43:55.858+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:55.858] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:55.858+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:43:55.858] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 170.607209ms. [2021-10-22T01:43:55.858+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:43:55.858] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38307 (local=false) [2021-10-22T01:43:55.858+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:43:55.858] Dropping existing documents from system.replset [2021-10-22T01:43:56.033+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:43:56.033] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:43:56.048+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:43:56.048] Inserting oplogTruncateAfterPoint document [2021-10-22T01:43:56.048+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:43:56.048] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:43:56.049+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:43:56.049] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:43:56.049+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:43:56.049] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:43:56.062+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:43:56.062] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:43:56.062+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:43:56.062] Setting the minvalid document to [{_id ObjectID("6172175c523370002becc648")} {t -1} {ts {0 1}}] [2021-10-22T01:43:56.074+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:43:56.074] Inserted [{_id ObjectID("6172175c523370002becc648")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:43:56.074+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:43:56.074] Inserting new replica set configuration document. [2021-10-22T01:43:56.074+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:43:56.074] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:43:56.086+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:43:56.086] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:43:56.090+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:43:56.090] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:43:56.090+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:43:56.090] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:43:56.090+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:43:56.090] Initiating stopEphemeralMongo [2021-10-22T01:43:56.090+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:43:56.090] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38307 (local=false) [2021-10-22T01:43:56.090+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:43:56.090] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38307 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:43:56.465+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:43:56.465] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.575697ms. [2021-10-22T01:43:56.465+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:43:56.465] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38307 (local=false) [2021-10-22T01:43:56.465+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:43:56.465] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.525µs. [2021-10-22T01:43:56.465+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:43:56.465] Successfully finished running stopEphemeralMongo [2021-10-22T01:43:56.465+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:43:56.465] Successfully stopped existing ephemeral mongod. [2021-10-22T01:43:56.465+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:43:56.465] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:43:56.465+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:43:56.465] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:43:56.515+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:43:56.515] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:43:56.516+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:43:56.516] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40579 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:43:56.805+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:43:56.805] clusterConfig unchanged [2021-10-22T01:43:56.863+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:56.863] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:57.706+0000] [.error] [src/action/start.go:func1:93] [103] [01:43:57.706] Error starting mongod : [01:43:57.706] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40579,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40579 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:43:56.547Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 1739 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:43:56.568+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:43:56.568+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:43:56.568+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40579,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:43:56.571+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:43:56.574+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:43:56.578+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:43:56.595+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:43:56.596+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:43:56.596+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:43:56.642+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:43:56.642+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:43:57.193+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867037:193904][1739:0x7f70a0ae7bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:43:57.300+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867037:300939][1739:0x7f70a0ae7bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:43:57.415+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867037:415307][1739:0x7f70a0ae7bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:43:57.530+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867037:530952][1739:0x7f70a0ae7bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:43:57.602+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867037:602820][1739:0x7f70a0ae7bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:43:57.658+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867037:658010][1739:0x7f70a0ae7bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:43:57.658+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867037:658083][1739:0x7f70a0ae7bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:43:57.707+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:43:57.707] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:40579 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:43:57.706] Error starting mongod : [01:43:57.706] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40579,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40579 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:43:56.547Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 1739 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:43:56.568+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:43:56.568+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:43:56.568+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40579,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:43:56.571+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:43:56.574+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:43:56.578+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:43:56.595+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:43:56.596+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:43:56.596+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:43:56.642+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:43:56.642+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:43:57.193+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867037:193904][1739:0x7f70a0ae7bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:43:57.300+00:00"},"s...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:43:57.707+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:43:57.707] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:43:57.707] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:40579 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:43:57.706] Error starting mongod : [01:43:57.706] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40579,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40579 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:43:56.547Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 1739 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:43:56.568+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:43:56.568+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:43:56.568+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40579,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:43:56.571+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:43:56.574+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:43:56.578+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:43:56.595+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:43:56.596+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:43:56.596+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:43:56.642+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:43:56.642+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:43:57.193+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867037:193904][1739:0x7f70a0ae7bc0... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:43:57.707+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:43:57.707] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:43:57.707+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:43:57.707] Initiating stopEphemeralMongo [2021-10-22T01:43:57.707+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:43:57.707] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:43:57.707+0000] [.error] [src/director/director.go:executePlan:962] [01:43:57.707] Failed to apply action. Result = : [01:43:57.707] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:43:57.707] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:40579 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:43:57.706] Error starting mongod : [01:43:57.706] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40579,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40579 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:43:56.547Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 1739 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:43:56.568+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:43:56.568+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:43:56.568+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40579,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:43:56.571+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:43:56.574+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:43:56.578+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:43:56.595+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:43:56.596+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:43:56.596+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:43:56.642+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:43:56.642+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:43:57.193+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:43:57.707+0000] [.error] [src/director/director.go:planAndExecute:567] [01:43:57.707] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:43:57.707] Failed to apply action. Result = : [01:43:57.707] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:43:57.707] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:40579 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:43:57.706] Error starting mongod : [01:43:57.706] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40579,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40579 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:43:56.547Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 1739 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:43:56.568+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:43:56.568+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:43:56.568+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40579,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:43:56.571+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:43:56.574+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:43:56.578+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:43:56.595+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:43:56.596+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:43:56.596+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:43:56.642+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:43:56.642+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:43:57.707+0000] [.error] [src/director/director.go:mainLoop:399] [01:43:57.707] Failed to planAndExecute : [01:43:57.707] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:43:57.707] Failed to apply action. Result = : [01:43:57.707] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:43:57.707] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:40579 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:43:57.706] Error starting mongod : [01:43:57.706] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40579,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40579 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:43:56.547Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 1739 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:43:56.568+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:43:56.568+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:43:56.568+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40579,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:43:56.571+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:43:56.574+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:43:56.578+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:43:56.595+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:43:56.596+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:43:56.596+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:43:56.642+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:43:56.642+00:00"},"s":"I", "c":"NETWORK", "id"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:43:57.864+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:57.864] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:58.035+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:43:58.035] clusterConfig unchanged [2021-10-22T01:43:58.880+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:58.880] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:43:59.864+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:43:59.864] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:00.866+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:00.866] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:01.866+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:01.866] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:02.867+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:02.867] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:03.866+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:03.866] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:04.867+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:04.867] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:05.865+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:05.865] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:06.678+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:44:06.678] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.583879446s [2021-10-22T01:44:06.865+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:06.865] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:07.865+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:07.865] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:08.138+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:44:08.138] clusterConfig unchanged [2021-10-22T01:44:08.872+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:08.872] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:09.866+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:09.866] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:10.867+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:10.867] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:11.867+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:11.867] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:12.867+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:12.867] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:13.868+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:13.868] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:14.869+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:14.869] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:15.869+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:15.869] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:16.870+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:16.870] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:17.873+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:17.873] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:18.279+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:44:18.279] clusterConfig unchanged [2021-10-22T01:44:18.871+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:18.871] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:19.872+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:19.872] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:20.873+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:20.873] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:21.873+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:21.873] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:22.874+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:22.874] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:23.874+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:23.874] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:24.875+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:24.875] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:25.875+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:25.875] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:26.876+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:26.876] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:27.876+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:27.876] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:28.421+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:44:28.421] clusterConfig unchanged [2021-10-22T01:44:28.877+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:28.877] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:29.878+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:29.878] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:30.885+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:30.885] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:31.880+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:31.880] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:32.881+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:32.881] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:33.882+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:33.882] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:34.882+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:34.882] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:35.883+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:35.883] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:36.882+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:36.882] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:37.883+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:37.883] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:38.522+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:44:38.522] clusterConfig unchanged [2021-10-22T01:44:38.883+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:38.883] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:39.883+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:39.883] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:40.884+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:40.884] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:41.349+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:44:41.349] memory:rss=54235136 alloc=13479024 totalalloc=7172213928 mallocs=91446470 frees=91331200 heapinuse=16973824 heapobjects=115270 gcpausetotalns=1533940971 gcpauselastns=47152 gcnum=1100 utimens=141493676000 stimens=51857120000 minflt=372597 majflt=405 [2021-10-22T01:44:41.885+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:41.885] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:42.885+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:42.885] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:43.885+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:43.885] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:44.314+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [01:44:44.314] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40579 (local=false) because it is not up [2021-10-22T01:44:44.886+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:44.885] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:45.886+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:45.886] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:46.887+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:46.887] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:47.887+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:47.887] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:48.625+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:44:48.625] clusterConfig unchanged [2021-10-22T01:44:48.888+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:48.888] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:49.889+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:49.889] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:50.890+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:50.890] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:51.889+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:51.889] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:52.896+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:52.896] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:53.890+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:53.890] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:54.890+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:54.890] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:55.892+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:55.891] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:56.892+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:56.892] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:57.871+0000] [.info] [src/director/director.go:computePlan:280] [01:44:57.871] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:44:57.871+0000] [.info] [src/director/director.go:tracef:794] [01:44:57.871] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:44:57.871+0000] [.info] [src/director/director.go:tracef:794] [01:44:57.871] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:44:57.871+0000] [.info] [src/director/director.go:tracef:794] [01:44:57.871] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:44:57.871+0000] [.info] [src/director/director.go:tracef:794] [01:44:57.871] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:44:57.871+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:44:57.871] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:44:57.893+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:57.893] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:58.034+0000] [.info] [src/director/director.go:tracef:794] [01:44:58.034] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:44:58.034+0000] [.info] [src/director/director.go:tracef:794] [01:44:58.034] because [2021-10-22T01:44:58.034+0000] [.info] [src/director/director.go:tracef:794] [01:44:58.034] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:44:58.034+0000] [.info] [src/director/director.go:tracef:794] [01:44:58.034] because ['currentState.CanBounceStop' = true] [2021-10-22T01:44:58.217+0000] [.info] [src/director/director.go:tracef:794] [01:44:58.217] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:44:58.217+0000] [.info] [src/director/director.go:tracef:794] [01:44:58.217] because [2021-10-22T01:44:58.217+0000] [.info] [src/director/director.go:tracef:794] [01:44:58.217] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:44:58.217+0000] [.info] [src/director/director.go:tracef:794] [01:44:58.217] because ['currentState.Up' = false] [2021-10-22T01:44:58.217+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:44:58.217] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:44:58.217+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:44:58.217] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:44:58.293+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:44:58.293] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:44:58.302+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:44:58.302] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:44:58.302+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:44:58.302] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:44:58.303+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:44:58.303] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:44:58.304+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:44:58.304] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:44:58.304+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:44:58.304] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:44:58.304+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:44:58.304] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:44:58.305+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:44:58.305] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:44:58.534+0000] [.info] [src/director/director.go:tracef:794] [01:44:58.534] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:44:58.535+0000] [.info] [src/director/director.go:tracef:794] [01:44:58.535] because ['currentState.Up' = false] [2021-10-22T01:44:58.741+0000] [.info] [src/director/director.go:tracef:794] [01:44:58.741] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:44:58.741+0000] [.info] [src/director/director.go:tracef:794] [01:44:58.741] because ['currentState.Up' = false] [2021-10-22T01:44:58.741+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:44:58.741] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:44:58.810+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:44:58.810] clusterConfig unchanged [2021-10-22T01:44:58.900+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:58.900] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:58.950+0000] [.info] [src/director/director.go:tracef:794] [01:44:58.950] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:44:58.950+0000] [.info] [src/director/director.go:tracef:794] [01:44:58.950] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:44:59.147+0000] [.info] [src/director/director.go:tracef:794] [01:44:59.147] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:44:59.147+0000] [.info] [src/director/director.go:tracef:794] [01:44:59.147] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:44:59.147+0000] [.info] [src/director/director.go:tracef:794] [01:44:59.147] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:44:59.147+0000] [.info] [src/director/director.go:tracef:794] [01:44:59.147] because ['currentState.Up' = false] [2021-10-22T01:44:59.147+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:44:59.147] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:44:59.147+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:44:59.147] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:44:59.182+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:44:59.182] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:44:59.183+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:44:59.183] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32982 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:44:59.898+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:44:59.898] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:44:59.904+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:44:59.904] ...Finished start [pid=20964] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32982,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:44:59.952+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:44:59.952] clusterConfig unchanged [2021-10-22T01:45:00.021+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:45:00.021] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 116.985508ms. [2021-10-22T01:45:00.021+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:45:00.021] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:32982 (local=false) [2021-10-22T01:45:00.021+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:45:00.021] Dropping existing documents from system.replset [2021-10-22T01:45:00.169+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:45:00.168] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:45:00.183+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:45:00.183] Inserting oplogTruncateAfterPoint document [2021-10-22T01:45:00.183+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:45:00.183] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:45:00.184+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:45:00.184] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:45:00.184+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:45:00.184] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:45:00.197+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:45:00.197] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:45:00.197+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:45:00.197] Setting the minvalid document to [{_id ObjectID("6172179c523370002becc657")} {t -1} {ts {0 1}}] [2021-10-22T01:45:00.211+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:45:00.211] Inserted [{_id ObjectID("6172179c523370002becc657")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:45:00.211+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:45:00.211] Inserting new replica set configuration document. [2021-10-22T01:45:00.211+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:45:00.211] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:45:00.224+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:45:00.224] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:45:00.227+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:45:00.227] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:45:00.227+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:45:00.227] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:45:00.227+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:45:00.227] Initiating stopEphemeralMongo [2021-10-22T01:45:00.227+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:45:00.227] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:32982 (local=false) [2021-10-22T01:45:00.227+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:45:00.227] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:32982 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:45:00.604+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:45:00.604] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.149817ms. [2021-10-22T01:45:00.604+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:45:00.604] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:32982 (local=false) [2021-10-22T01:45:00.604+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:45:00.604] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.268µs. [2021-10-22T01:45:00.604+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:45:00.604] Successfully finished running stopEphemeralMongo [2021-10-22T01:45:00.604+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:45:00.604] Successfully stopped existing ephemeral mongod. [2021-10-22T01:45:00.604+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:45:00.604] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:45:00.604+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:45:00.604] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:45:00.639+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:45:00.639] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:45:00.640+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:45:00.640] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46496 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:45:00.903+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:00.903] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:01.090+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:45:01.090] clusterConfig unchanged [2021-10-22T01:45:01.923+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:01.923] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:01.975+0000] [.error] [src/action/start.go:func1:93] [103] [01:45:01.974] Error starting mongod : [01:45:01.974] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46496,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46496 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 22412 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:45:00.695+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:45:00.695+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:45:00.695+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46496,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:45:00.696+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:45:00.698+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:45:00.698+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:45:00.712+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:45:00.712+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:45:00.720+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:45:00.765+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:45:00.765+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:45:01.305+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867101:305789][22412:0x7fa94b727bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:45:01.465+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867101:465709][22412:0x7fa94b727bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:45:01.621+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867101:621007][22412:0x7fa94b727bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:45:01.754+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867101:754120][22412:0x7fa94b727bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:45:01.827+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867101:827729][22412:0x7fa94b727bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:45:01.910+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867101:910909][22412:0x7fa94b727bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:45:01.910+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867101:910995][22412:0x7fa94b727bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:45:01.923+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:45:01.975+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:45:01.975] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46496 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:45:01.974] Error starting mongod : [01:45:01.974] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46496,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46496 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 22412 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:45:00.695+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:45:00.695+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:45:00.695+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46496,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:45:00.696+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:45:00.698+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:45:00.698+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:45:00.712+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:45:00.712+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:45:00.720+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:45:00.765+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:45:00.765+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:45:01.305+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867101:305789][22412:0x7fa94b727bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:45:01.465+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867101:465709][22412:0x7fa94b727bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:45:01.975+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:45:01.975] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:45:01.975] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46496 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:45:01.974] Error starting mongod : [01:45:01.974] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46496,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46496 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 22412 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:45:00.695+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:45:00.695+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:45:00.695+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46496,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:45:00.696+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:45:00.698+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:45:00.698+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:45:00.712+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:45:00.712+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:45:00.720+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:45:00.765+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:45:00.765+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:45:01.305+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867101:305789][22412:0x7fa94b727bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:45:01.465+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:45:01.975+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:45:01.975] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:45:01.975+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:45:01.975] Initiating stopEphemeralMongo [2021-10-22T01:45:01.975+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:45:01.975] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:45:01.976+0000] [.error] [src/director/director.go:executePlan:962] [01:45:01.976] Failed to apply action. Result = : [01:45:01.975] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:45:01.975] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46496 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:45:01.974] Error starting mongod : [01:45:01.974] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46496,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46496 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 22412 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:45:00.695+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:45:00.695+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:45:00.695+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46496,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:45:00.696+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:45:00.698+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:45:00.698+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:45:00.712+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:45:00.712+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:45:00.720+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:45:00.765+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:45:00.765+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:45:01.305+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867101:305789][22412:0x7fa94b727bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:45:01.976+0000] [.error] [src/director/director.go:planAndExecute:567] [01:45:01.976] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:45:01.976] Failed to apply action. Result = : [01:45:01.975] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:45:01.975] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46496 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:45:01.974] Error starting mongod : [01:45:01.974] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46496,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46496 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 22412 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:45:00.695+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:45:00.695+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:45:00.695+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46496,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:45:00.696+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:45:00.698+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:45:00.698+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:45:00.712+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:45:00.712+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:45:00.720+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:45:00.765+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:45:00.765+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:45:01.305+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:45:01.976+0000] [.error] [src/director/director.go:mainLoop:399] [01:45:01.976] Failed to planAndExecute : [01:45:01.976] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:45:01.976] Failed to apply action. Result = : [01:45:01.975] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:45:01.975] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:46496 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:45:01.974] Error starting mongod : [01:45:01.974] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46496,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46496 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 22412 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:45:00.695+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:45:00.695+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:45:00.695+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46496,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:45:00.696+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:45:00.698+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:45:00.698+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:45:00.712+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:45:00.712+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:45:00.720+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:45:00.765+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:45:00.765+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:45:01.305+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:45:02.320+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:45:02.320] clusterConfig unchanged [2021-10-22T01:45:02.903+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:02.903] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:03.904+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:03.904] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:04.904+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:04.904] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:05.910+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:05.910] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:06.912+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:06.912] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:07.909+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:07.909] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:08.910+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:08.910] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:09.912+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:09.912] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:10.910+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:10.910] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:11.029+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:45:11.029] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.724152424s [2021-10-22T01:45:11.909+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:11.909] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:12.442+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:45:12.442] clusterConfig unchanged [2021-10-22T01:45:12.909+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:12.909] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:13.910+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:13.910] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:14.910+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:14.910] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:15.910+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:15.910] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:16.913+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:16.913] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:17.911+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:17.911] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:18.911+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:18.911] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:19.917+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:19.917] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:20.912+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:20.912] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:21.913+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:21.913] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:22.564+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:45:22.564] clusterConfig unchanged [2021-10-22T01:45:22.913+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:22.913] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:23.916+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:23.916] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:24.914+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:24.914] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:25.914+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:25.914] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:26.915+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:26.915] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:27.916+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:27.916] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:28.917+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:28.917] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:29.917+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:29.917] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:30.920+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:30.920] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:31.925+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:31.925] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:32.785+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:45:32.785] clusterConfig unchanged [2021-10-22T01:45:32.918+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:32.918] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:33.919+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:33.919] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:34.919+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:34.919] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:35.922+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:35.922] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:36.920+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:36.920] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:37.920+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:37.920] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:38.921+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:38.921] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:39.922+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:39.922] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:40.922+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:40.922] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:41.350+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:45:41.350] memory:rss=54439936 alloc=13231704 totalalloc=7261603552 mallocs=92585274 frees=92473982 heapinuse=17285120 heapobjects=111292 gcpausetotalns=1535968060 gcpauselastns=46201 gcnum=1112 utimens=143613397000 stimens=52802334000 minflt=377312 majflt=405 [2021-10-22T01:45:41.923+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:41.923] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:42.924+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:42.924] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:42.946+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:45:42.946] clusterConfig unchanged [2021-10-22T01:45:43.925+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:43.925] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:44.925+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:44.925] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:45.926+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:45.926] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:46.927+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:46.927] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:47.927+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:47.927] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:48.927+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:48.927] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:49.928+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:49.928] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:50.932+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:50.932] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:51.930+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:51.930] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:52.931+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:52.931] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:53.058+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:45:53.058] clusterConfig unchanged [2021-10-22T01:45:53.931+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:53.931] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:54.938+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:54.938] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:55.932+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:55.932] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:56.933+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:56.933] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:57.933+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:57.933] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:58.934+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:58.934] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:45:59.934+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:45:59.934] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:00.935+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:00.935] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:01.936+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:01.936] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:02.216+0000] [.info] [src/director/director.go:computePlan:280] [01:46:02.216] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:46:02.216+0000] [.info] [src/director/director.go:tracef:794] [01:46:02.216] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:46:02.216+0000] [.info] [src/director/director.go:tracef:794] [01:46:02.216] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:46:02.216+0000] [.info] [src/director/director.go:tracef:794] [01:46:02.216] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:46:02.216+0000] [.info] [src/director/director.go:tracef:794] [01:46:02.216] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:46:02.217+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:46:02.217] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:46:02.402+0000] [.info] [src/director/director.go:tracef:794] [01:46:02.402] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:46:02.402+0000] [.info] [src/director/director.go:tracef:794] [01:46:02.402] because [2021-10-22T01:46:02.403+0000] [.info] [src/director/director.go:tracef:794] [01:46:02.403] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:46:02.403+0000] [.info] [src/director/director.go:tracef:794] [01:46:02.403] because ['currentState.CanBounceStop' = true] [2021-10-22T01:46:02.580+0000] [.info] [src/director/director.go:tracef:794] [01:46:02.580] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:46:02.580+0000] [.info] [src/director/director.go:tracef:794] [01:46:02.580] because [2021-10-22T01:46:02.580+0000] [.info] [src/director/director.go:tracef:794] [01:46:02.580] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:46:02.580+0000] [.info] [src/director/director.go:tracef:794] [01:46:02.580] because ['currentState.Up' = false] [2021-10-22T01:46:02.580+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:46:02.580] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:46:02.580+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:46:02.580] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:46:02.766+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:46:02.766] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:46:02.775+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:46:02.775] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:46:02.775+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:46:02.775] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:46:02.776+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:46:02.776] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:46:02.777+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:46:02.777] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:46:02.777+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:46:02.777] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:46:02.777+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:46:02.777] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:46:02.779+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:46:02.779] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:46:02.940+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:02.940] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:02.998+0000] [.info] [src/director/director.go:tracef:794] [01:46:02.998] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:46:02.998+0000] [.info] [src/director/director.go:tracef:794] [01:46:02.998] because ['currentState.Up' = false] [2021-10-22T01:46:03.237+0000] [.info] [src/director/director.go:tracef:794] [01:46:03.237] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:46:03.237+0000] [.info] [src/director/director.go:tracef:794] [01:46:03.237] because ['currentState.Up' = false] [2021-10-22T01:46:03.237+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:46:03.237] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:46:03.288+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:46:03.288] clusterConfig unchanged [2021-10-22T01:46:03.467+0000] [.info] [src/director/director.go:tracef:794] [01:46:03.467] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:46:03.467+0000] [.info] [src/director/director.go:tracef:794] [01:46:03.467] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:46:03.686+0000] [.info] [src/director/director.go:tracef:794] [01:46:03.686] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:46:03.686+0000] [.info] [src/director/director.go:tracef:794] [01:46:03.686] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:46:03.686+0000] [.info] [src/director/director.go:tracef:794] [01:46:03.686] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:46:03.686+0000] [.info] [src/director/director.go:tracef:794] [01:46:03.686] because ['currentState.Up' = false] [2021-10-22T01:46:03.686+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:46:03.686] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:46:03.686+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:46:03.686] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:46:03.717+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:46:03.717] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:46:03.717+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:46:03.717] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42527 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:46:03.958+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:03.958] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:04.422+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:46:04.422] clusterConfig unchanged [2021-10-22T01:46:04.444+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:46:04.444] ...Finished start [pid=41734] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42527,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:46:04.578+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:46:04.578] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 134.553859ms. [2021-10-22T01:46:04.578+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:46:04.578] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42527 (local=false) [2021-10-22T01:46:04.578+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:46:04.578] Dropping existing documents from system.replset [2021-10-22T01:46:04.698+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:46:04.698] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:46:04.703+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:46:04.703] Inserting oplogTruncateAfterPoint document [2021-10-22T01:46:04.703+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:46:04.703] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:46:04.704+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:46:04.704] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:46:04.704+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:46:04.704] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:46:04.718+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:46:04.718] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:46:04.718+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:46:04.718] Setting the minvalid document to [{_id ObjectID("617217dc523370002becc665")} {t -1} {ts {0 1}}] [2021-10-22T01:46:04.731+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:46:04.731] Inserted [{_id ObjectID("617217dc523370002becc665")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:46:04.731+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:46:04.731] Inserting new replica set configuration document. [2021-10-22T01:46:04.731+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:46:04.731] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:46:04.744+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:46:04.744] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:46:04.748+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:46:04.748] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:46:04.748+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:46:04.748] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:46:04.748+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:46:04.748] Initiating stopEphemeralMongo [2021-10-22T01:46:04.748+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:46:04.748] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42527 (local=false) [2021-10-22T01:46:04.748+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:46:04.748] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42527 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:46:04.950+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:04.950] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:05.131+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:46:05.131] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.995679ms. [2021-10-22T01:46:05.131+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:46:05.131] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42527 (local=false) [2021-10-22T01:46:05.131+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:46:05.131] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.627µs. [2021-10-22T01:46:05.131+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:46:05.131] Successfully finished running stopEphemeralMongo [2021-10-22T01:46:05.131+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:46:05.131] Successfully stopped existing ephemeral mongod. [2021-10-22T01:46:05.131+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:46:05.131] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:46:05.131+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:46:05.131] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:46:05.166+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:46:05.166] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:46:05.167+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:46:05.167] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53583 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:46:05.527+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:46:05.527] clusterConfig unchanged [2021-10-22T01:46:05.952+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:05.952] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:06.328+0000] [.error] [src/action/start.go:func1:93] [103] [01:46:06.328] Error starting mongod : [01:46:06.328] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53583,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53583 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 43216 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:46:05.219+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:46:05.219+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:46:05.219+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53583,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:46:05.221+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:46:05.222+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:46:05.222+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:46:05.300+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:46:05.300+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:46:05.316+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:46:05.370+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:46:05.370+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:46:05.820+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867165:820285][43216:0x7ff00f676bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:46:05.921+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867165:921529][43216:0x7ff00f676bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:46:06.024+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867166:24498][43216:0x7ff00f676bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:46:06.141+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867166:141675][43216:0x7ff00f676bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:46:06.219+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867166:219256][43216:0x7ff00f676bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:46:06.277+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867166:277744][43216:0x7ff00f676bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:46:06.277+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867166:277833][43216:0x7ff00f676bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:46:06.279+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:46:06.328+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:46:06.328] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53583 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:46:06.328] Error starting mongod : [01:46:06.328] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53583,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53583 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 43216 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:46:05.219+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:46:05.219+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:46:05.219+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53583,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:46:05.221+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:46:05.222+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:46:05.222+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:46:05.300+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:46:05.300+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:46:05.316+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:46:05.370+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:46:05.370+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:46:05.820+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867165:820285][43216:0x7ff00f676bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:46:05.921+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867165:921529][43216:0x7ff00f676bc0], txn-recover: [W...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:46:06.328+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:46:06.328] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:46:06.328] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53583 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:46:06.328] Error starting mongod : [01:46:06.328] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53583,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53583 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 43216 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:46:05.219+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:46:05.219+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:46:05.219+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53583,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:46:05.221+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:46:05.222+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:46:05.222+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:46:05.300+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:46:05.300+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:46:05.316+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:46:05.370+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:46:05.370+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:46:05.820+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867165:820285][43216:0x7ff00f676bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:46:05.921+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ct... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:46:06.329+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:46:06.328] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:46:06.329+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:46:06.329] Initiating stopEphemeralMongo [2021-10-22T01:46:06.329+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:46:06.329] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:46:06.329+0000] [.error] [src/director/director.go:executePlan:962] [01:46:06.329] Failed to apply action. Result = : [01:46:06.328] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:46:06.328] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53583 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:46:06.328] Error starting mongod : [01:46:06.328] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53583,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53583 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 43216 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:46:05.219+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:46:05.219+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:46:05.219+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53583,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:46:05.221+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:46:05.222+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:46:05.222+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:46:05.300+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:46:05.300+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:46:05.316+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:46:05.370+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:46:05.370+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:46:05.820+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867165:820285][43216:0x7ff00f676bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:46:06.329+0000] [.error] [src/director/director.go:planAndExecute:567] [01:46:06.329] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:46:06.329] Failed to apply action. Result = : [01:46:06.328] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:46:06.328] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53583 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:46:06.328] Error starting mongod : [01:46:06.328] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53583,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53583 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 43216 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:46:05.219+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:46:05.219+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:46:05.219+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53583,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:46:05.221+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:46:05.222+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:46:05.222+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:46:05.300+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:46:05.300+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:46:05.316+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:46:05.370+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:46:05.370+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:46:05.820+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:46:06.329+0000] [.error] [src/director/director.go:mainLoop:399] [01:46:06.329] Failed to planAndExecute : [01:46:06.329] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:46:06.329] Failed to apply action. Result = : [01:46:06.328] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:46:06.328] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:53583 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:46:06.328] Error starting mongod : [01:46:06.328] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53583,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53583 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 43216 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:46:05.219+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:46:05.219+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:46:05.219+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53583,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:46:05.221+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:46:05.222+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:46:05.222+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:46:05.300+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:46:05.300+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:46:05.316+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:46:05.370+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:46:05.370+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:46:05.820+00:00"},"s":"I", "c":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:46:06.774+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:46:06.774] clusterConfig unchanged [2021-10-22T01:46:06.952+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:06.952] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:07.953+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:07.953] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:08.968+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:08.968] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:09.953+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:09.953] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:10.954+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:10.954] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:11.954+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:11.954] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:12.955+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:12.955] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:13.953+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:13.953] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:14.957+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:14.957] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:15.408+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:46:15.408] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.629685541s [2021-10-22T01:46:15.954+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:15.954] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:16.955+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:16.955] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:17.004+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:46:17.003] clusterConfig unchanged [2021-10-22T01:46:17.956+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:17.956] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:18.563+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:46:18.563] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:46:18.571+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:18.571] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:46:18.571+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [01:46:18.571] Could not get version : [01:46:18.571] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:46:18.571+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [01:46:18.571] Error getting version for log lines. Skipping : [01:46:18.571] Could not get version : [01:46:18.571] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:46:18.957+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:18.957] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:19.957+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:19.957] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:20.958+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:20.958] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:21.958+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:21.958] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:22.959+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:22.959] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:23.959+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:23.959] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:24.960+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:24.960] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:25.960+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:25.960] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:26.961+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:26.961] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:27.106+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:46:27.106] clusterConfig unchanged [2021-10-22T01:46:27.961+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:27.961] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:28.962+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:28.962] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:29.970+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:29.969] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:30.964+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:30.964] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:31.971+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:31.971] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:32.965+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:32.965] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:33.966+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:33.965] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:34.966+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:34.966] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:35.967+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:35.967] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:36.967+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:36.967] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:37.208+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:46:37.208] clusterConfig unchanged [2021-10-22T01:46:37.968+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:37.968] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:38.968+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:38.968] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:39.969+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:39.969] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:40.978+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:40.978] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:41.351+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:46:41.351] memory:rss=55119872 alloc=14445632 totalalloc=7352520872 mallocs=93734513 frees=93608584 heapinuse=17629184 heapobjects=125929 gcpausetotalns=1537617983 gcpauselastns=136155 gcnum=1124 utimens=145618767000 stimens=53682383000 minflt=382181 majflt=405 [2021-10-22T01:46:41.971+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:41.970] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:42.971+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:42.971] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:43.971+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:43.971] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:44.972+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:44.972] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:45.973+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:45.973] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:46.973+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:46.973] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:47.330+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:46:47.330] clusterConfig unchanged [2021-10-22T01:46:47.974+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:47.974] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:48.975+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:48.975] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:49.977+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:49.977] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:50.975+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:50.975] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:51.976+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:51.976] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:52.977+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:52.977] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:53.977+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:53.977] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:54.978+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:54.978] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:55.979+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:55.979] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:56.979+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:56.979] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:57.453+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:46:57.453] clusterConfig unchanged [2021-10-22T01:46:57.980+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:57.980] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:58.985+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:58.985] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:46:59.981+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:46:59.981] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:00.982+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:00.982] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:01.983+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:01.983] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:02.984+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:02.984] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:03.985+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:03.985] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:04.986+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:04.986] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:05.988+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:05.988] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:06.514+0000] [.info] [src/director/director.go:computePlan:280] [01:47:06.514] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:47:06.514+0000] [.info] [src/director/director.go:tracef:794] [01:47:06.514] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:47:06.514+0000] [.info] [src/director/director.go:tracef:794] [01:47:06.514] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:47:06.514+0000] [.info] [src/director/director.go:tracef:794] [01:47:06.514] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:47:06.514+0000] [.info] [src/director/director.go:tracef:794] [01:47:06.514] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:47:06.514+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:47:06.514] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:47:06.700+0000] [.info] [src/director/director.go:tracef:794] [01:47:06.700] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:47:06.700+0000] [.info] [src/director/director.go:tracef:794] [01:47:06.700] because [2021-10-22T01:47:06.700+0000] [.info] [src/director/director.go:tracef:794] [01:47:06.700] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:47:06.700+0000] [.info] [src/director/director.go:tracef:794] [01:47:06.700] because ['currentState.CanBounceStop' = true] [2021-10-22T01:47:06.853+0000] [.info] [src/director/director.go:tracef:794] [01:47:06.853] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:47:06.853+0000] [.info] [src/director/director.go:tracef:794] [01:47:06.853] because [2021-10-22T01:47:06.853+0000] [.info] [src/director/director.go:tracef:794] [01:47:06.853] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:47:06.853+0000] [.info] [src/director/director.go:tracef:794] [01:47:06.853] because ['currentState.Up' = false] [2021-10-22T01:47:06.853+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:47:06.853] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:47:06.853+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:47:06.853] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:47:06.974+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:47:06.974] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:47:06.983+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:47:06.983] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:47:06.985+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:47:06.985] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:47:06.986+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:47:06.986] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:47:06.990+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:47:06.990] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:47:06.990+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:47:06.990] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:47:06.992+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:47:06.992] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:47:06.993+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:47:06.993] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:47:07.004+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:07.004] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:07.228+0000] [.info] [src/director/director.go:tracef:794] [01:47:07.228] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:47:07.228+0000] [.info] [src/director/director.go:tracef:794] [01:47:07.228] because ['currentState.Up' = false] [2021-10-22T01:47:07.519+0000] [.info] [src/director/director.go:tracef:794] [01:47:07.519] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:47:07.520+0000] [.info] [src/director/director.go:tracef:794] [01:47:07.520] because ['currentState.Up' = false] [2021-10-22T01:47:07.520+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:47:07.520] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:47:07.642+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:47:07.642] clusterConfig unchanged [2021-10-22T01:47:07.728+0000] [.info] [src/director/director.go:tracef:794] [01:47:07.728] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:47:07.728+0000] [.info] [src/director/director.go:tracef:794] [01:47:07.728] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:47:07.933+0000] [.info] [src/director/director.go:tracef:794] [01:47:07.933] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:47:07.933+0000] [.info] [src/director/director.go:tracef:794] [01:47:07.933] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:47:07.934+0000] [.info] [src/director/director.go:tracef:794] [01:47:07.934] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:47:07.934+0000] [.info] [src/director/director.go:tracef:794] [01:47:07.934] because ['currentState.Up' = false] [2021-10-22T01:47:07.934+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:47:07.934] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:47:07.934+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:47:07.934] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:47:07.969+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:47:07.969] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:47:07.970+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:47:07.970] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41473 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:47:07.992+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:07.992] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:08.704+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:47:08.704] ...Finished start [pid=62569] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41473,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:47:08.771+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:47:08.771] clusterConfig unchanged [2021-10-22T01:47:08.826+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:47:08.826] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 122.090459ms. [2021-10-22T01:47:08.826+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:47:08.826] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41473 (local=false) [2021-10-22T01:47:08.827+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:47:08.827] Dropping existing documents from system.replset [2021-10-22T01:47:08.977+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:47:08.977] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:47:08.990+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:08.990] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:08.992+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:47:08.992] Inserting oplogTruncateAfterPoint document [2021-10-22T01:47:08.992+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:47:08.992] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:47:08.994+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:47:08.994] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:47:08.994+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:47:08.994] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:47:09.006+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:47:09.006] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:47:09.006+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:47:09.006] Setting the minvalid document to [{_id ObjectID("6172181d523370002becc673")} {t -1} {ts {0 1}}] [2021-10-22T01:47:09.018+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:47:09.018] Inserted [{_id ObjectID("6172181d523370002becc673")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:47:09.018+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:47:09.018] Inserting new replica set configuration document. [2021-10-22T01:47:09.018+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:47:09.018] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:47:09.030+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:47:09.030] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:47:09.034+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:47:09.034] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:47:09.034+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:47:09.034] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:47:09.034+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:47:09.034] Initiating stopEphemeralMongo [2021-10-22T01:47:09.034+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:47:09.034] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41473 (local=false) [2021-10-22T01:47:09.034+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:47:09.034] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41473 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:47:09.458+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:47:09.458] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.121117ms. [2021-10-22T01:47:09.458+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:47:09.458] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41473 (local=false) [2021-10-22T01:47:09.458+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:47:09.458] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 33.71µs. [2021-10-22T01:47:09.458+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:47:09.458] Successfully finished running stopEphemeralMongo [2021-10-22T01:47:09.458+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:47:09.458] Successfully stopped existing ephemeral mongod. [2021-10-22T01:47:09.458+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:47:09.458] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:47:09.458+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:47:09.458] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:47:09.493+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:47:09.493] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:47:09.493+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:47:09.493] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32782 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:47:09.858+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:47:09.858] clusterConfig unchanged [2021-10-22T01:47:09.991+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:09.991] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:10.707+0000] [.error] [src/action/start.go:func1:93] [103] [01:47:10.707] Error starting mongod : [01:47:10.707] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32782,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32782 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 64079 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:47:09.545+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:47:09.545+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:47:09.545+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32782,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:47:09.546+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:47:09.548+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:47:09.549+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:47:09.612+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:47:09.612+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:47:09.612+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:47:09.658+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:47:09.658+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:47:10.160+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867230:160462][64079:0x7f1669b28bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:47:10.282+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867230:282367][64079:0x7f1669b28bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:47:10.388+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867230:388602][64079:0x7f1669b28bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T01:47:10.505+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867230:505900][64079:0x7f1669b28bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:47:10.585+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867230:585764][64079:0x7f1669b28bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:47:10.644+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867230:644731][64079:0x7f1669b28bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:47:10.644+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867230:644800][64079:0x7f1669b28bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:47:10.646+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:47:10.707+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:47:10.707] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:32782 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:47:10.707] Error starting mongod : [01:47:10.707] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32782,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32782 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 64079 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:47:09.545+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:47:09.545+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:47:09.545+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32782,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:47:09.546+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:47:09.548+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:47:09.549+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:47:09.612+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:47:09.612+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:47:09.612+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:47:09.658+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:47:09.658+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:47:10.160+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867230:160462][64079:0x7f1669b28bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:47:10.282+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867230:282367][64079:0x7f1669b28bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:47:10.708+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:47:10.708] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:47:10.707] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:32782 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:47:10.707] Error starting mongod : [01:47:10.707] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32782,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32782 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 64079 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:47:09.545+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:47:09.545+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:47:09.545+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32782,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:47:09.546+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:47:09.548+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:47:09.549+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:47:09.612+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:47:09.612+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:47:09.612+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:47:09.658+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:47:09.658+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:47:10.160+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867230:160462][64079:0x7f1669b28bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:47:10.282+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:47:10.708+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:47:10.708] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:47:10.708+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:47:10.708] Initiating stopEphemeralMongo [2021-10-22T01:47:10.708+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:47:10.708] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:47:10.708+0000] [.error] [src/director/director.go:executePlan:962] [01:47:10.708] Failed to apply action. Result = : [01:47:10.708] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:47:10.707] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:32782 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:47:10.707] Error starting mongod : [01:47:10.707] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32782,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32782 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 64079 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:47:09.545+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:47:09.545+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:47:09.545+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32782,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:47:09.546+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:47:09.548+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:47:09.549+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:47:09.612+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:47:09.612+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:47:09.612+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:47:09.658+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:47:09.658+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:47:10.160+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867230:160462][64079:0x7f1669b28bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:47:10.709+0000] [.error] [src/director/director.go:planAndExecute:567] [01:47:10.708] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:47:10.708] Failed to apply action. Result = : [01:47:10.708] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:47:10.707] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:32782 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:47:10.707] Error starting mongod : [01:47:10.707] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32782,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32782 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 64079 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:47:09.545+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:47:09.545+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:47:09.545+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32782,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:47:09.546+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:47:09.548+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:47:09.549+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:47:09.612+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:47:09.612+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:47:09.612+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:47:09.658+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:47:09.658+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:47:10.160+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:47:10.709+0000] [.error] [src/director/director.go:mainLoop:399] [01:47:10.709] Failed to planAndExecute : [01:47:10.708] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:47:10.708] Failed to apply action. Result = : [01:47:10.708] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:47:10.707] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:32782 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:47:10.707] Error starting mongod : [01:47:10.707] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32782,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32782 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 64079 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:47:09.545+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:47:09.545+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:47:09.545+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32782,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:47:09.546+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:47:09.548+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:47:09.549+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:47:09.612+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:47:09.612+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:47:09.612+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:47:09.658+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:47:09.658+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:47:10.160+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:47:10.991+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:10.991] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:11.129+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:47:11.129] clusterConfig unchanged [2021-10-22T01:47:11.988+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:11.988] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:12.991+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:12.991] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:13.992+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:13.992] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:15.004+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:15.004] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:16.005+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:16.005] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:17.004+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:17.004] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:18.005+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:18.005] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:19.007+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:19.007] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:19.662+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:47:19.662] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.668096552s [2021-10-22T01:47:20.004+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:20.004] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:21.004+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:21.004] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:21.261+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:47:21.261] clusterConfig unchanged [2021-10-22T01:47:22.012+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:22.012] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:23.006+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:23.006] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:24.007+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:24.007] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:25.007+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:25.007] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:26.007+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:26.007] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:27.008+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:27.008] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:27.566+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:47:27.566] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:47:27.567+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:47:27.567] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:47:28.010+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:28.010] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:29.010+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:29.010] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:30.010+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:30.010] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:31.011+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:31.011] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:31.372+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:47:31.372] clusterConfig unchanged [2021-10-22T01:47:32.011+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:32.011] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:33.017+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:33.017] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:34.011+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:34.011] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:35.012+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:35.012] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:36.013+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:36.013] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:37.015+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [01:47:37.015] lsblk cache invalidated, running lsblk command [2021-10-22T01:47:37.016+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:37.016] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:38.014+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:38.014] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:39.015+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:39.015] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:40.015+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:40.015] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:41.015+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:41.015] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:41.352+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:47:41.352] memory:rss=54259712 alloc=12322760 totalalloc=7442747496 mallocs=94884661 frees=94782452 heapinuse=16629760 heapobjects=102209 gcpausetotalns=1547448022 gcpauselastns=90832 gcnum=1137 utimens=147849211000 stimens=54600447000 minflt=387537 majflt=405 [2021-10-22T01:47:41.535+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:47:41.535] clusterConfig unchanged [2021-10-22T01:47:42.015+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:42.015] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:43.022+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:43.022] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:44.017+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:44.017] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:44.322+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [01:47:44.322] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:32782 (local=false) because it is not up [2021-10-22T01:47:45.030+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:45.030] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:46.019+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:46.019] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:47.019+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:47.018] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:48.019+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:48.019] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:49.019+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:49.019] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:50.020+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:50.020] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:51.021+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:51.021] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:51.636+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:47:51.636] clusterConfig unchanged [2021-10-22T01:47:52.021+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:52.021] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:53.028+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:53.028] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:54.023+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:54.023] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:55.023+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:55.023] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:56.023+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:56.023] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:57.024+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:57.024] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:58.025+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:58.025] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:47:59.027+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:47:59.027] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:00.026+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:00.026] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:01.028+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:01.028] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:01.738+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:48:01.738] clusterConfig unchanged [2021-10-22T01:48:02.028+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:02.028] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:03.027+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:03.027] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:04.035+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:04.035] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:05.029+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:05.029] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:06.030+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:06.030] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:07.029+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:07.029] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:08.030+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:08.030] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:09.031+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:09.031] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:10.032+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:10.032] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:10.883+0000] [.info] [src/director/director.go:computePlan:280] [01:48:10.883] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:48:10.883+0000] [.info] [src/director/director.go:tracef:794] [01:48:10.883] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:48:10.883+0000] [.info] [src/director/director.go:tracef:794] [01:48:10.883] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:48:10.883+0000] [.info] [src/director/director.go:tracef:794] [01:48:10.883] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:48:10.883+0000] [.info] [src/director/director.go:tracef:794] [01:48:10.883] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:48:10.883+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:48:10.883] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:48:11.035+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:11.035] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:11.042+0000] [.info] [src/director/director.go:tracef:794] [01:48:11.042] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:48:11.042+0000] [.info] [src/director/director.go:tracef:794] [01:48:11.042] because [2021-10-22T01:48:11.042+0000] [.info] [src/director/director.go:tracef:794] [01:48:11.042] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:48:11.042+0000] [.info] [src/director/director.go:tracef:794] [01:48:11.042] because ['currentState.CanBounceStop' = true] [2021-10-22T01:48:11.194+0000] [.info] [src/director/director.go:tracef:794] [01:48:11.194] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:48:11.194+0000] [.info] [src/director/director.go:tracef:794] [01:48:11.194] because [2021-10-22T01:48:11.194+0000] [.info] [src/director/director.go:tracef:794] [01:48:11.194] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:48:11.194+0000] [.info] [src/director/director.go:tracef:794] [01:48:11.194] because ['currentState.Up' = false] [2021-10-22T01:48:11.194+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:48:11.194] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:48:11.194+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:48:11.194] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:48:11.279+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:48:11.279] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:48:11.287+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:48:11.287] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:48:11.288+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:48:11.288] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:48:11.289+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:48:11.289] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:48:11.289+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:48:11.289] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:48:11.290+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:48:11.290] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:48:11.290+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:48:11.290] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:48:11.291+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:48:11.291] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:48:11.530+0000] [.info] [src/director/director.go:tracef:794] [01:48:11.530] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:48:11.530+0000] [.info] [src/director/director.go:tracef:794] [01:48:11.530] because ['currentState.Up' = false] [2021-10-22T01:48:11.753+0000] [.info] [src/director/director.go:tracef:794] [01:48:11.753] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:48:11.753+0000] [.info] [src/director/director.go:tracef:794] [01:48:11.753] because ['currentState.Up' = false] [2021-10-22T01:48:11.753+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:48:11.753] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:48:11.943+0000] [.info] [src/director/director.go:tracef:794] [01:48:11.943] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:48:11.943+0000] [.info] [src/director/director.go:tracef:794] [01:48:11.943] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:48:11.959+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:48:11.959] clusterConfig unchanged [2021-10-22T01:48:12.042+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:12.042] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:12.169+0000] [.info] [src/director/director.go:tracef:794] [01:48:12.168] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:48:12.169+0000] [.info] [src/director/director.go:tracef:794] [01:48:12.169] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:48:12.169+0000] [.info] [src/director/director.go:tracef:794] [01:48:12.169] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:48:12.169+0000] [.info] [src/director/director.go:tracef:794] [01:48:12.169] because ['currentState.Up' = false] [2021-10-22T01:48:12.169+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:48:12.169] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:48:12.169+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:48:12.169] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:48:12.199+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:48:12.199] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:48:12.200+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:48:12.200] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52001 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:48:12.916+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:48:12.916] ...Finished start [pid=9240] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52001,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:48:13.037+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:13.037] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:13.038+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:48:13.038] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 122.22062ms. [2021-10-22T01:48:13.038+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:48:13.038] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52001 (local=false) [2021-10-22T01:48:13.038+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:48:13.038] Dropping existing documents from system.replset [2021-10-22T01:48:13.131+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:48:13.131] clusterConfig unchanged [2021-10-22T01:48:13.183+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:48:13.183] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:48:13.187+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:48:13.187] Inserting oplogTruncateAfterPoint document [2021-10-22T01:48:13.187+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:48:13.187] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:48:13.188+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:48:13.188] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:48:13.188+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:48:13.188] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:48:13.201+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:48:13.201] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:48:13.201+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:48:13.201] Setting the minvalid document to [{_id ObjectID("6172185d523370002becc681")} {t -1} {ts {0 1}}] [2021-10-22T01:48:13.213+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:48:13.212] Inserted [{_id ObjectID("6172185d523370002becc681")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:48:13.213+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:48:13.213] Inserting new replica set configuration document. [2021-10-22T01:48:13.213+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:48:13.213] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:48:13.226+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:48:13.226] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:48:13.230+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:48:13.230] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:48:13.230+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:48:13.230] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:48:13.230+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:48:13.230] Initiating stopEphemeralMongo [2021-10-22T01:48:13.230+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:48:13.230] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52001 (local=false) [2021-10-22T01:48:13.230+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:48:13.230] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52001 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:48:13.614+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:48:13.614] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.313145ms. [2021-10-22T01:48:13.614+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:48:13.614] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:52001 (local=false) [2021-10-22T01:48:13.614+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:48:13.614] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.091µs. [2021-10-22T01:48:13.614+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:48:13.614] Successfully finished running stopEphemeralMongo [2021-10-22T01:48:13.614+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:48:13.614] Successfully stopped existing ephemeral mongod. [2021-10-22T01:48:13.614+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:48:13.614] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:48:13.614+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:48:13.614] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:48:13.650+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:48:13.650] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:48:13.650+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:48:13.650] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38143 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:48:14.039+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:14.039] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:14.252+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:48:14.252] clusterConfig unchanged [2021-10-22T01:48:14.801+0000] [.error] [src/action/start.go:func1:93] [103] [01:48:14.801] Error starting mongod : [01:48:14.801] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38143,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38143 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 10792 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:48:13.701+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:48:13.701+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:48:13.701+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38143,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:48:13.703+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:48:13.705+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:48:13.705+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:48:13.732+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:48:13.732+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:48:13.733+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:48:13.786+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:48:13.786+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:48:14.312+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867294:312703][10792:0x7f196ae9ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:48:14.415+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867294:415618][10792:0x7f196ae9ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:48:14.515+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867294:515738][10792:0x7f196ae9ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35456 to 2/256"}} {"t":{"$date":"2021-10-22T01:48:14.626+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867294:626927][10792:0x7f196ae9ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:48:14.699+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867294:699010][10792:0x7f196ae9ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:48:14.753+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867294:753680][10792:0x7f196ae9ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:48:14.753+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867294:753754][10792:0x7f196ae9ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:48:14.755+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:48:14.801+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:48:14.801] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38143 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:48:14.801] Error starting mongod : [01:48:14.801] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38143,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38143 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 10792 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:48:13.701+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:48:13.701+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:48:13.701+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38143,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:48:13.703+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:48:13.705+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:48:13.705+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:48:13.732+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:48:13.732+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:48:13.733+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:48:13.786+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:48:13.786+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:48:14.312+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867294:312703][10792:0x7f196ae9ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:48:14.415+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867294:415618][10792:0x7f196ae9ebc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:48:14.801+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:48:14.801] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:48:14.801] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38143 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:48:14.801] Error starting mongod : [01:48:14.801] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38143,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38143 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 10792 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:48:13.701+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:48:13.701+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:48:13.701+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38143,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:48:13.703+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:48:13.705+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:48:13.705+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:48:13.732+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:48:13.732+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:48:13.733+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:48:13.786+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:48:13.786+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:48:14.312+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867294:312703][10792:0x7f196ae9ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:48:14.415+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:48:14.802+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:48:14.802] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:48:14.802+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:48:14.802] Initiating stopEphemeralMongo [2021-10-22T01:48:14.802+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:48:14.802] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:48:14.802+0000] [.error] [src/director/director.go:executePlan:962] [01:48:14.802] Failed to apply action. Result = : [01:48:14.801] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:48:14.801] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38143 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:48:14.801] Error starting mongod : [01:48:14.801] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38143,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38143 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 10792 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:48:13.701+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:48:13.701+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:48:13.701+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38143,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:48:13.703+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:48:13.705+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:48:13.705+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:48:13.732+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:48:13.732+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:48:13.733+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:48:13.786+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:48:13.786+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:48:14.312+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867294:312703][10792:0x7f196ae9ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:48:14.802+0000] [.error] [src/director/director.go:planAndExecute:567] [01:48:14.802] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:48:14.802] Failed to apply action. Result = : [01:48:14.801] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:48:14.801] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38143 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:48:14.801] Error starting mongod : [01:48:14.801] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38143,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38143 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 10792 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:48:13.701+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:48:13.701+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:48:13.701+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38143,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:48:13.703+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:48:13.705+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:48:13.705+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:48:13.732+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:48:13.732+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:48:13.733+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:48:13.786+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:48:13.786+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:48:14.312+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:48:14.802+0000] [.error] [src/director/director.go:mainLoop:399] [01:48:14.802] Failed to planAndExecute : [01:48:14.802] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:48:14.802] Failed to apply action. Result = : [01:48:14.801] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:48:14.801] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38143 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:48:14.801] Error starting mongod : [01:48:14.801] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38143,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38143 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 10792 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:48:13.701+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:48:13.701+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:48:13.701+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38143,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:48:13.703+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:48:13.705+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:48:13.705+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:48:13.732+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:48:13.732+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:48:13.733+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:48:13.786+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:48:13.786+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:48:14.312+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:48:15.039+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:15.039] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:15.514+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:48:15.514] clusterConfig unchanged [2021-10-22T01:48:16.040+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:16.040] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:17.040+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:17.040] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:18.041+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:18.041] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:19.057+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:19.057] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:20.042+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:20.042] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:21.042+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:21.042] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:22.043+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:22.043] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:23.044+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:23.044] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:23.725+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:48:23.725] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.433784514s [2021-10-22T01:48:24.044+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:24.044] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:25.044+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:25.044] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:25.654+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:48:25.654] clusterConfig unchanged [2021-10-22T01:48:26.045+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:26.045] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:27.044+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:27.044] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:28.047+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:28.047] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:29.045+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:29.045] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:30.047+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:30.047] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:31.048+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:31.048] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:32.047+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:32.047] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:33.048+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:33.048] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:34.048+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:34.048] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:35.048+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:35.048] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:35.776+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:48:35.776] clusterConfig unchanged [2021-10-22T01:48:36.049+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:36.049] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:37.047+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:37.047] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:38.050+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:38.050] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:39.050+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:39.050] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:40.050+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:40.050] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:41.048+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:41.048] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:41.353+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:48:41.353] memory:rss=53387264 alloc=8043512 totalalloc=7533287104 mallocs=96035850 frees=95980911 heapinuse=15343616 heapobjects=54939 gcpausetotalns=1551395049 gcpauselastns=1090680 gcnum=1150 utimens=150164457000 stimens=55662727000 minflt=392425 majflt=405 [2021-10-22T01:48:42.052+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:42.052] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:43.056+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:43.056] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:44.050+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:44.050] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:45.051+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:45.051] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:45.907+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:48:45.907] clusterConfig unchanged [2021-10-22T01:48:46.065+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:46.065] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:47.053+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:47.052] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:48.054+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:48.054] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:49.059+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:49.059] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:50.054+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:50.054] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:51.071+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:51.071] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:52.056+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:52.056] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:53.056+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:53.056] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:54.059+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:54.059] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:55.058+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:55.058] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:56.049+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:48:56.049] clusterConfig unchanged [2021-10-22T01:48:56.060+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:56.060] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:57.062+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:57.062] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:58.059+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:58.059] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:48:59.060+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:48:59.060] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:00.075+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:00.075] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:01.077+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:01.077] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:02.062+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:02.062] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:03.062+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:03.062] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:04.063+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:04.063] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:05.063+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:05.063] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:06.063+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:06.063] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:06.181+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:49:06.181] clusterConfig unchanged [2021-10-22T01:49:07.067+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:07.067] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:08.068+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:08.068] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:09.065+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:09.065] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:10.066+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:10.066] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:11.066+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:11.066] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:12.067+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:12.067] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:13.066+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:13.066] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:14.067+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:14.067] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:14.965+0000] [.info] [src/director/director.go:computePlan:280] [01:49:14.965] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:49:14.966+0000] [.info] [src/director/director.go:tracef:794] [01:49:14.966] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:49:14.966+0000] [.info] [src/director/director.go:tracef:794] [01:49:14.966] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:49:14.966+0000] [.info] [src/director/director.go:tracef:794] [01:49:14.966] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:49:14.966+0000] [.info] [src/director/director.go:tracef:794] [01:49:14.966] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:49:14.966+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:49:14.966] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:49:15.069+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:15.069] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:15.131+0000] [.info] [src/director/director.go:tracef:794] [01:49:15.131] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:49:15.131+0000] [.info] [src/director/director.go:tracef:794] [01:49:15.131] because [2021-10-22T01:49:15.131+0000] [.info] [src/director/director.go:tracef:794] [01:49:15.131] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:49:15.131+0000] [.info] [src/director/director.go:tracef:794] [01:49:15.131] because ['currentState.CanBounceStop' = true] [2021-10-22T01:49:15.279+0000] [.info] [src/director/director.go:tracef:794] [01:49:15.279] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:49:15.279+0000] [.info] [src/director/director.go:tracef:794] [01:49:15.279] because [2021-10-22T01:49:15.279+0000] [.info] [src/director/director.go:tracef:794] [01:49:15.279] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:49:15.279+0000] [.info] [src/director/director.go:tracef:794] [01:49:15.279] because ['currentState.Up' = false] [2021-10-22T01:49:15.279+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:49:15.279] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:49:15.279+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:49:15.279] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:49:15.357+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:49:15.357] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:49:15.365+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:49:15.365] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:49:15.365+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:49:15.365] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:49:15.366+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:49:15.366] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:49:15.367+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:49:15.367] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:49:15.367+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:49:15.367] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:49:15.367+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:49:15.367] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:49:15.368+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:49:15.368] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:49:15.594+0000] [.info] [src/director/director.go:tracef:794] [01:49:15.594] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:49:15.594+0000] [.info] [src/director/director.go:tracef:794] [01:49:15.594] because ['currentState.Up' = false] [2021-10-22T01:49:15.805+0000] [.info] [src/director/director.go:tracef:794] [01:49:15.805] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:49:15.805+0000] [.info] [src/director/director.go:tracef:794] [01:49:15.805] because ['currentState.Up' = false] [2021-10-22T01:49:15.805+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:49:15.805] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:49:16.010+0000] [.info] [src/director/director.go:tracef:794] [01:49:16.010] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:49:16.010+0000] [.info] [src/director/director.go:tracef:794] [01:49:16.010] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:49:16.090+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:16.090] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:16.231+0000] [.info] [src/director/director.go:tracef:794] [01:49:16.231] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:49:16.231+0000] [.info] [src/director/director.go:tracef:794] [01:49:16.231] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:49:16.231+0000] [.info] [src/director/director.go:tracef:794] [01:49:16.231] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:49:16.231+0000] [.info] [src/director/director.go:tracef:794] [01:49:16.231] because ['currentState.Up' = false] [2021-10-22T01:49:16.232+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:49:16.231] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:49:16.232+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:49:16.232] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:49:16.268+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:49:16.268] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:49:16.268+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:49:16.268] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41892 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:49:16.391+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:49:16.391] clusterConfig unchanged [2021-10-22T01:49:17.020+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:49:17.020] ...Finished start [pid=30527] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41892,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:49:17.082+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:17.082] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:17.172+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:49:17.172] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 152.119575ms. [2021-10-22T01:49:17.172+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:49:17.172] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41892 (local=false) [2021-10-22T01:49:17.172+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:49:17.172] Dropping existing documents from system.replset [2021-10-22T01:49:17.373+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:49:17.373] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:49:17.377+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:49:17.376] Inserting oplogTruncateAfterPoint document [2021-10-22T01:49:17.377+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:49:17.377] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:49:17.378+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:49:17.378] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:49:17.378+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:49:17.378] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:49:17.392+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:49:17.392] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:49:17.392+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:49:17.392] Setting the minvalid document to [{_id ObjectID("6172189d523370002becc68f")} {t -1} {ts {0 1}}] [2021-10-22T01:49:17.405+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:49:17.405] Inserted [{_id ObjectID("6172189d523370002becc68f")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:49:17.405+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:49:17.405] Inserting new replica set configuration document. [2021-10-22T01:49:17.405+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:49:17.405] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:49:17.419+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:49:17.419] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:49:17.434+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:49:17.434] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:49:17.434+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:49:17.434] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:49:17.434+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:49:17.434] Initiating stopEphemeralMongo [2021-10-22T01:49:17.434+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:49:17.434] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41892 (local=false) [2021-10-22T01:49:17.434+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:49:17.434] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41892 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:49:17.523+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:49:17.523] clusterConfig unchanged [2021-10-22T01:49:17.855+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:49:17.855] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.106187ms. [2021-10-22T01:49:17.855+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:49:17.855] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41892 (local=false) [2021-10-22T01:49:17.855+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:49:17.855] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.498µs. [2021-10-22T01:49:17.855+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:49:17.855] Successfully finished running stopEphemeralMongo [2021-10-22T01:49:17.855+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:49:17.855] Successfully stopped existing ephemeral mongod. [2021-10-22T01:49:17.855+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:49:17.855] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:49:17.855+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:49:17.855] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:49:17.890+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:49:17.890] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:49:17.891+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:49:17.891] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49130 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:49:18.083+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:18.083] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:18.636+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:49:18.636] clusterConfig unchanged [2021-10-22T01:49:19.080+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:19.080] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:19.107+0000] [.error] [src/action/start.go:func1:93] [103] [01:49:19.107] Error starting mongod : [01:49:19.106] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49130,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49130 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:49:17.923Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 32022 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:49:17.955+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:49:17.955+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:49:17.955+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49130,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:49:17.956+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:49:17.958+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:49:17.961+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:49:18.001+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:49:18.002+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:49:18.004+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:49:18.051+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:49:18.051+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:49:18.563+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867358:563428][32022:0x7efe09d80bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:49:18.707+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867358:707727][32022:0x7efe09d80bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:49:18.810+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867358:810532][32022:0x7efe09d80bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:49:18.926+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867358:926278][32022:0x7efe09d80bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:49:18.998+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867358:998738][32022:0x7efe09d80bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:49:19.056+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867359:56156][32022:0x7efe09d80bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:49:19.056+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867359:56231][32022:0x7efe09d80bc0], txn-recover: [WT_VERB_RECOVERY | WT_...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:49:19.107+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:49:19.107] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49130 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:49:19.107] Error starting mongod : [01:49:19.106] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49130,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49130 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:49:17.923Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 32022 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:49:17.955+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:49:17.955+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:49:17.955+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49130,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:49:17.956+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:49:17.958+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:49:17.961+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:49:18.001+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:49:18.002+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:49:18.004+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:49:18.051+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:49:18.051+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:49:18.563+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867358:563428][32022:0x7efe09d80bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:49:18.707+00:00"},"s":"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:49:19.107+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:49:19.107] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:49:19.107] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49130 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:49:19.107] Error starting mongod : [01:49:19.106] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49130,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49130 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:49:17.923Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 32022 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:49:17.955+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:49:17.955+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:49:17.955+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49130,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:49:17.956+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:49:17.958+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:49:17.961+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:49:18.001+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:49:18.002+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:49:18.004+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:49:18.051+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:49:18.051+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:49:18.563+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867358:563428][32022:0x7efe09d80bc0], ... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:49:19.107+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:49:19.107] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:49:19.107+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:49:19.107] Initiating stopEphemeralMongo [2021-10-22T01:49:19.107+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:49:19.107] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:49:19.108+0000] [.error] [src/director/director.go:executePlan:962] [01:49:19.108] Failed to apply action. Result = : [01:49:19.107] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:49:19.107] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49130 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:49:19.107] Error starting mongod : [01:49:19.106] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49130,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49130 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:49:17.923Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 32022 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:49:17.955+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:49:17.955+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:49:17.955+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49130,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:49:17.956+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:49:17.958+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:49:17.961+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:49:18.001+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:49:18.002+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:49:18.004+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:49:18.051+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:49:18.051+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:49:18.563+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","m... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:49:19.108+0000] [.error] [src/director/director.go:planAndExecute:567] [01:49:19.108] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:49:19.108] Failed to apply action. Result = : [01:49:19.107] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:49:19.107] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49130 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:49:19.107] Error starting mongod : [01:49:19.106] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49130,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49130 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:49:17.923Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 32022 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:49:17.955+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:49:17.955+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:49:17.955+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49130,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:49:17.956+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:49:17.958+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:49:17.961+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:49:18.001+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:49:18.002+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:49:18.004+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:49:18.051+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:49:18.051+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:49:19.108+0000] [.error] [src/director/director.go:mainLoop:399] [01:49:19.108] Failed to planAndExecute : [01:49:19.108] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:49:19.108] Failed to apply action. Result = : [01:49:19.107] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:49:19.107] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49130 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:49:19.107] Error starting mongod : [01:49:19.106] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49130,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49130 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:49:17.923Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 32022 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:49:17.955+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:49:17.955+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:49:17.955+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49130,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:49:17.956+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:49:17.958+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:49:17.961+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:49:18.001+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:49:18.002+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:49:18.004+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:49:18.051+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:49:18.051+00:00"},"s":"I", "c":"NETWORK", "id":577... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:49:19.876+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:49:19.876] clusterConfig unchanged [2021-10-22T01:49:20.084+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:20.084] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:21.084+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:21.084] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:22.086+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:22.086] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:23.083+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:23.083] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:24.086+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:24.086] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:25.086+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:25.086] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:26.085+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:26.085] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:27.088+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:27.088] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:28.088+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:28.088] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:28.151+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:49:28.151] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.782304805s [2021-10-22T01:49:29.087+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:29.087] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:29.978+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:49:29.978] clusterConfig unchanged [2021-10-22T01:49:30.087+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:30.087] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:31.090+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:31.090] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:32.088+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:32.088] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:33.096+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:33.096] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:34.089+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:34.089] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:35.090+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:35.090] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:36.090+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:36.090] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:37.091+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:37.091] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:38.092+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:38.092] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:39.092+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:39.092] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:40.071+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:49:40.071] clusterConfig unchanged [2021-10-22T01:49:40.093+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:40.093] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:41.100+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:41.100] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:41.353+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:49:41.353] memory:rss=54947840 alloc=12735568 totalalloc=7622938568 mallocs=97176077 frees=97069455 heapinuse=17014784 heapobjects=106622 gcpausetotalns=1552998225 gcpauselastns=164398 gcnum=1162 utimens=152485125000 stimens=56716815000 minflt=397919 majflt=405 [2021-10-22T01:49:42.095+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:42.095] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:43.095+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:43.095] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:44.095+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:44.095] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:45.097+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:45.097] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:46.097+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:46.097] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:47.098+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:47.098] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:48.099+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:48.099] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:49.100+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:49.100] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:50.115+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:50.115] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:50.193+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:49:50.193] clusterConfig unchanged [2021-10-22T01:49:51.101+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:51.101] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:52.101+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:52.101] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:53.101+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:53.101] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:54.102+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:54.102] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:55.109+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:55.109] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:56.104+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:56.104] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:57.104+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:57.104] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:58.105+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:58.105] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:49:59.105+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:49:59.105] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:00.106+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:00.106] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:00.345+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:50:00.345] clusterConfig unchanged [2021-10-22T01:50:01.106+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:01.106] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:02.107+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:02.107] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:03.107+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:03.107] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:04.107+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:04.107] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:05.108+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:05.108] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:06.108+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:06.108] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:07.109+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:07.109] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:08.109+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:08.109] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:09.109+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:09.109] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:10.110+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:10.110] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:10.467+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:50:10.467] clusterConfig unchanged [2021-10-22T01:50:11.111+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:11.111] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:12.111+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:12.111] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:13.112+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:13.112] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:14.124+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:14.124] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:15.114+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:15.114] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:16.119+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:16.119] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:17.114+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:17.114] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:18.115+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:18.115] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:19.116+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:19.116] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:19.270+0000] [.info] [src/director/director.go:computePlan:280] [01:50:19.270] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:50:19.270+0000] [.info] [src/director/director.go:tracef:794] [01:50:19.270] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:50:19.271+0000] [.info] [src/director/director.go:tracef:794] [01:50:19.270] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:50:19.271+0000] [.info] [src/director/director.go:tracef:794] [01:50:19.271] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:50:19.271+0000] [.info] [src/director/director.go:tracef:794] [01:50:19.271] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:50:19.271+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:50:19.271] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:50:19.436+0000] [.info] [src/director/director.go:tracef:794] [01:50:19.436] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:50:19.436+0000] [.info] [src/director/director.go:tracef:794] [01:50:19.436] because [2021-10-22T01:50:19.436+0000] [.info] [src/director/director.go:tracef:794] [01:50:19.436] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:50:19.436+0000] [.info] [src/director/director.go:tracef:794] [01:50:19.436] because ['currentState.CanBounceStop' = true] [2021-10-22T01:50:19.607+0000] [.info] [src/director/director.go:tracef:794] [01:50:19.607] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:50:19.607+0000] [.info] [src/director/director.go:tracef:794] [01:50:19.607] because [2021-10-22T01:50:19.607+0000] [.info] [src/director/director.go:tracef:794] [01:50:19.607] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:50:19.607+0000] [.info] [src/director/director.go:tracef:794] [01:50:19.607] because ['currentState.Up' = false] [2021-10-22T01:50:19.607+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:50:19.607] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:50:19.607+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:50:19.607] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:50:19.692+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:50:19.692] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:50:19.693+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:50:19.693] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:50:19.701+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:50:19.701] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:50:19.703+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:50:19.703] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:50:19.703+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:50:19.703] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:50:19.703+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:50:19.703] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:50:19.705+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:50:19.705] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:50:19.943+0000] [.info] [src/director/director.go:tracef:794] [01:50:19.943] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:50:19.944+0000] [.info] [src/director/director.go:tracef:794] [01:50:19.944] because ['currentState.Up' = false] [2021-10-22T01:50:20.119+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:20.119] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:20.158+0000] [.info] [src/director/director.go:tracef:794] [01:50:20.158] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:50:20.158+0000] [.info] [src/director/director.go:tracef:794] [01:50:20.158] because ['currentState.Up' = false] [2021-10-22T01:50:20.158+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:50:20.158] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:50:20.366+0000] [.info] [src/director/director.go:tracef:794] [01:50:20.366] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:50:20.366+0000] [.info] [src/director/director.go:tracef:794] [01:50:20.366] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:50:20.626+0000] [.info] [src/director/director.go:tracef:794] [01:50:20.626] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:50:20.626+0000] [.info] [src/director/director.go:tracef:794] [01:50:20.626] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:50:20.627+0000] [.info] [src/director/director.go:tracef:794] [01:50:20.627] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:50:20.627+0000] [.info] [src/director/director.go:tracef:794] [01:50:20.627] because ['currentState.Up' = false] [2021-10-22T01:50:20.627+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:50:20.627] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:50:20.627+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:50:20.627] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:50:20.661+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:50:20.661] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:50:20.662+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:50:20.662] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37856 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:50:20.737+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:50:20.737] clusterConfig unchanged [2021-10-22T01:50:21.125+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:21.125] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:21.389+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:50:21.389] ...Finished start [pid=51340] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37856,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:50:21.514+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:50:21.514] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 123.967872ms. [2021-10-22T01:50:21.514+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:50:21.514] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37856 (local=false) [2021-10-22T01:50:21.514+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:50:21.514] Dropping existing documents from system.replset [2021-10-22T01:50:21.708+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:50:21.708] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:50:21.711+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:50:21.711] Inserting oplogTruncateAfterPoint document [2021-10-22T01:50:21.711+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:50:21.711] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:50:21.712+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:50:21.712] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:50:21.712+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:50:21.712] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:50:21.724+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:50:21.724] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:50:21.725+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:50:21.725] Setting the minvalid document to [{_id ObjectID("617218dd523370002becc69d")} {t -1} {ts {0 1}}] [2021-10-22T01:50:21.736+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:50:21.736] Inserted [{_id ObjectID("617218dd523370002becc69d")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:50:21.736+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:50:21.736] Inserting new replica set configuration document. [2021-10-22T01:50:21.736+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:50:21.736] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:50:21.749+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:50:21.749] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:50:21.753+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:50:21.753] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:50:21.753+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:50:21.753] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:50:21.753+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:50:21.753] Initiating stopEphemeralMongo [2021-10-22T01:50:21.753+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:50:21.753] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37856 (local=false) [2021-10-22T01:50:21.753+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:50:21.753] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37856 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:50:21.853+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:50:21.853] clusterConfig unchanged [2021-10-22T01:50:22.124+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:22.124] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:22.196+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:50:22.196] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.602637ms. [2021-10-22T01:50:22.196+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:50:22.196] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37856 (local=false) [2021-10-22T01:50:22.197+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:50:22.196] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.918µs. [2021-10-22T01:50:22.197+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:50:22.197] Successfully finished running stopEphemeralMongo [2021-10-22T01:50:22.197+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:50:22.197] Successfully stopped existing ephemeral mongod. [2021-10-22T01:50:22.197+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:50:22.197] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:50:22.197+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:50:22.197] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:50:22.232+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:50:22.232] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:50:22.235+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:50:22.235] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37797 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:50:22.959+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:50:22.959] clusterConfig unchanged [2021-10-22T01:50:23.127+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:23.127] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:23.444+0000] [.error] [src/action/start.go:func1:93] [103] [01:50:23.444] Error starting mongod : [01:50:23.444] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37797,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37797 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:50:22.266Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 52900 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:50:22.288+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:50:22.288+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:50:22.288+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37797,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:50:22.289+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:50:22.290+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:50:22.291+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:50:22.313+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:50:22.313+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:50:22.314+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:50:22.370+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:50:22.370+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:50:22.931+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867422:931124][52900:0x7fd9a7954bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:50:23.037+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867423:37156][52900:0x7fd9a7954bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:50:23.144+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867423:144940][52900:0x7fd9a7954bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35456 to 2/256"}} {"t":{"$date":"2021-10-22T01:50:23.259+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867423:259367][52900:0x7fd9a7954bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:50:23.331+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867423:331157][52900:0x7fd9a7954bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:50:23.394+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867423:394712][52900:0x7fd9a7954bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:50:23.394+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867423:394792][52900:0x7fd9a7954bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:50:23.444+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:50:23.444] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37797 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:50:23.444] Error starting mongod : [01:50:23.444] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37797,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37797 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:50:22.266Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 52900 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:50:22.288+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:50:22.288+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:50:22.288+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37797,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:50:22.289+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:50:22.290+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:50:22.291+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:50:22.313+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:50:22.313+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:50:22.314+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:50:22.370+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:50:22.370+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:50:22.931+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867422:931124][52900:0x7fd9a7954bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:50:23.037+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:50:23.444+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:50:23.444] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:50:23.444] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37797 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:50:23.444] Error starting mongod : [01:50:23.444] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37797,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37797 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:50:22.266Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 52900 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:50:22.288+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:50:22.288+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:50:22.288+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37797,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:50:22.289+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:50:22.290+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:50:22.291+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:50:22.313+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:50:22.313+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:50:22.314+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:50:22.370+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:50:22.370+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:50:22.931+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867422:931124][52900:0x7fd9a7954bc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:50:23.444+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:50:23.444] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:50:23.445+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:50:23.444] Initiating stopEphemeralMongo [2021-10-22T01:50:23.445+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:50:23.445] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:50:23.445+0000] [.error] [src/director/director.go:executePlan:962] [01:50:23.445] Failed to apply action. Result = : [01:50:23.444] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:50:23.444] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37797 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:50:23.444] Error starting mongod : [01:50:23.444] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37797,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37797 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:50:22.266Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 52900 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:50:22.288+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:50:22.288+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:50:22.288+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37797,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:50:22.289+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:50:22.290+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:50:22.291+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:50:22.313+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:50:22.313+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:50:22.314+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:50:22.370+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:50:22.370+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:50:22.931+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:50:23.445+0000] [.error] [src/director/director.go:planAndExecute:567] [01:50:23.445] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:50:23.445] Failed to apply action. Result = : [01:50:23.444] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:50:23.444] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37797 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:50:23.444] Error starting mongod : [01:50:23.444] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37797,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37797 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:50:22.266Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 52900 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:50:22.288+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:50:22.288+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:50:22.288+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37797,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:50:22.289+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:50:22.290+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:50:22.291+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:50:22.313+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:50:22.313+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:50:22.314+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:50:22.370+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:50:22.370+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:50:23.445+0000] [.error] [src/director/director.go:mainLoop:399] [01:50:23.445] Failed to planAndExecute : [01:50:23.445] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:50:23.445] Failed to apply action. Result = : [01:50:23.444] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:50:23.444] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37797 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:50:23.444] Error starting mongod : [01:50:23.444] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37797,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37797 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:50:22.266Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 52900 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:50:22.288+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:50:22.288+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:50:22.288+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37797,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:50:22.289+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:50:22.290+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:50:22.291+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:50:22.313+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:50:22.313+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:50:22.314+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:50:22.370+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:50:22.370+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:50:24.126+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:24.126] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:24.221+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:50:24.221] clusterConfig unchanged [2021-10-22T01:50:25.127+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:25.127] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:26.127+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:26.126] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:27.128+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:27.128] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:28.129+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:28.129] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:29.130+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:29.129] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:30.130+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:30.130] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:31.130+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:31.130] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:32.130+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:32.130] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:32.437+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:50:32.437] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.731741423s [2021-10-22T01:50:33.129+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:33.129] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:34.129+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:34.129] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:34.333+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:50:34.333] clusterConfig unchanged [2021-10-22T01:50:35.130+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:35.130] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:36.130+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:36.130] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:37.130+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:37.130] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:38.132+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:38.132] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:39.133+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:39.133] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:40.139+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:40.139] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:41.133+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:41.133] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:41.354+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:50:41.354] memory:rss=53747712 alloc=8511448 totalalloc=7712667008 mallocs=98318127 frees=98259834 heapinuse=15982592 heapobjects=58293 gcpausetotalns=1554492207 gcpauselastns=58869 gcnum=1175 utimens=154744800000 stimens=57657024000 minflt=402803 majflt=405 [2021-10-22T01:50:42.134+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:42.134] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:43.134+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:43.134] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:44.134+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:44.134] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:44.330+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [01:50:44.330] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37797 (local=false) because it is not up [2021-10-22T01:50:44.435+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:50:44.435] clusterConfig unchanged [2021-10-22T01:50:45.135+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:45.135] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:46.136+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:46.136] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:47.136+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:47.136] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:48.143+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:48.143] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:49.137+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:49.137] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:50.137+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:50.137] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:51.139+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:51.139] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:52.139+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:52.139] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:53.140+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:53.140] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:54.140+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:54.140] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:54.547+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:50:54.547] clusterConfig unchanged [2021-10-22T01:50:55.140+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:55.140] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:56.140+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:56.140] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:57.140+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:57.140] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:58.141+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:58.141] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:50:59.147+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:50:59.147] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:00.143+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:00.143] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:01.143+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:01.143] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:02.144+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:02.144] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:03.145+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:03.144] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:04.145+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:04.145] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:04.679+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:51:04.679] clusterConfig unchanged [2021-10-22T01:51:05.146+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:05.146] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:06.147+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:06.147] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:07.148+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:07.148] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:08.148+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:08.148] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:09.148+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:09.148] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:10.151+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:10.151] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:11.150+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:11.150] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:12.150+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:12.150] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:13.151+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:13.151] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:14.151+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:14.151] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:14.889+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:51:14.889] clusterConfig unchanged [2021-10-22T01:51:15.151+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:15.151] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:16.152+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:16.152] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:17.153+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:17.153] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:18.153+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:18.153] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:18.563+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:51:18.563] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:51:18.571+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:18.571] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:51:18.571+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [01:51:18.571] Could not get version : [01:51:18.571] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:51:18.571+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [01:51:18.571] Error getting version for log lines. Skipping : [01:51:18.571] Could not get version : [01:51:18.571] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:51:19.154+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:19.154] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:20.155+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:20.155] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:21.155+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:21.155] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:22.155+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:22.155] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:23.155+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:23.155] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:23.607+0000] [.info] [src/director/director.go:computePlan:280] [01:51:23.607] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:51:23.607+0000] [.info] [src/director/director.go:tracef:794] [01:51:23.607] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:51:23.607+0000] [.info] [src/director/director.go:tracef:794] [01:51:23.607] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:51:23.607+0000] [.info] [src/director/director.go:tracef:794] [01:51:23.607] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:51:23.607+0000] [.info] [src/director/director.go:tracef:794] [01:51:23.607] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:51:23.607+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:51:23.607] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:51:23.764+0000] [.info] [src/director/director.go:tracef:794] [01:51:23.764] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:51:23.764+0000] [.info] [src/director/director.go:tracef:794] [01:51:23.764] because [2021-10-22T01:51:23.764+0000] [.info] [src/director/director.go:tracef:794] [01:51:23.764] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:51:23.764+0000] [.info] [src/director/director.go:tracef:794] [01:51:23.764] because ['currentState.CanBounceStop' = true] [2021-10-22T01:51:23.910+0000] [.info] [src/director/director.go:tracef:794] [01:51:23.910] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:51:23.910+0000] [.info] [src/director/director.go:tracef:794] [01:51:23.910] because [2021-10-22T01:51:23.910+0000] [.info] [src/director/director.go:tracef:794] [01:51:23.910] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:51:23.910+0000] [.info] [src/director/director.go:tracef:794] [01:51:23.910] because ['currentState.Up' = false] [2021-10-22T01:51:23.910+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:51:23.910] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:51:23.910+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:51:23.910] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:51:24.095+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:51:24.095] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:51:24.105+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:51:24.105] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:51:24.106+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:51:24.106] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:51:24.107+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:51:24.107] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:51:24.107+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:51:24.107] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:51:24.108+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:51:24.108] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:51:24.108+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:51:24.108] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:51:24.109+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:51:24.109] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:51:24.173+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:24.173] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:24.356+0000] [.info] [src/director/director.go:tracef:794] [01:51:24.356] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:51:24.356+0000] [.info] [src/director/director.go:tracef:794] [01:51:24.356] because ['currentState.Up' = false] [2021-10-22T01:51:24.579+0000] [.info] [src/director/director.go:tracef:794] [01:51:24.579] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:51:24.579+0000] [.info] [src/director/director.go:tracef:794] [01:51:24.579] because ['currentState.Up' = false] [2021-10-22T01:51:24.579+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:51:24.579] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:51:24.779+0000] [.info] [src/director/director.go:tracef:794] [01:51:24.779] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:51:24.779+0000] [.info] [src/director/director.go:tracef:794] [01:51:24.779] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:51:24.964+0000] [.info] [src/director/director.go:tracef:794] [01:51:24.964] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:51:24.964+0000] [.info] [src/director/director.go:tracef:794] [01:51:24.964] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:51:24.965+0000] [.info] [src/director/director.go:tracef:794] [01:51:24.965] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:51:24.965+0000] [.info] [src/director/director.go:tracef:794] [01:51:24.965] because ['currentState.Up' = false] [2021-10-22T01:51:24.965+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:51:24.965] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:51:24.965+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:51:24.965] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:51:25.001+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:51:25.001] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:51:25.002+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:51:25.002] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39343 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:51:25.071+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:51:25.070] clusterConfig unchanged [2021-10-22T01:51:25.161+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:25.161] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:25.743+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:51:25.743] ...Finished start [pid=71992] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39343,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:51:25.879+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:51:25.879] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 136.613271ms. [2021-10-22T01:51:25.880+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:51:25.880] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39343 (local=false) [2021-10-22T01:51:25.880+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:51:25.880] Dropping existing documents from system.replset [2021-10-22T01:51:25.998+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:51:25.998] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:51:26.003+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:51:26.002] Inserting oplogTruncateAfterPoint document [2021-10-22T01:51:26.003+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:51:26.003] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:51:26.004+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:51:26.004] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:51:26.004+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:51:26.004] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:51:26.019+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:51:26.019] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:51:26.019+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:51:26.019] Setting the minvalid document to [{_id ObjectID("6172191e523370002becc6ab")} {t -1} {ts {0 1}}] [2021-10-22T01:51:26.031+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:51:26.031] Inserted [{_id ObjectID("6172191e523370002becc6ab")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:51:26.031+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:51:26.031] Inserting new replica set configuration document. [2021-10-22T01:51:26.031+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:51:26.031] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:51:26.044+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:51:26.044] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:51:26.055+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:51:26.055] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:51:26.055+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:51:26.055] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:51:26.055+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:51:26.055] Initiating stopEphemeralMongo [2021-10-22T01:51:26.055+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:51:26.055] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39343 (local=false) [2021-10-22T01:51:26.055+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:51:26.055] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39343 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:51:26.164+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:26.164] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:26.193+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:51:26.193] clusterConfig unchanged [2021-10-22T01:51:26.440+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:51:26.440] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.155695ms. [2021-10-22T01:51:26.440+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:51:26.440] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39343 (local=false) [2021-10-22T01:51:26.440+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:51:26.440] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 40.283µs. [2021-10-22T01:51:26.440+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:51:26.440] Successfully finished running stopEphemeralMongo [2021-10-22T01:51:26.440+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:51:26.440] Successfully stopped existing ephemeral mongod. [2021-10-22T01:51:26.440+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:51:26.440] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:51:26.440+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:51:26.440] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:51:26.487+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:51:26.487] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:51:26.488+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:51:26.488] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39630 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:51:27.164+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:27.164] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:27.325+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:51:27.325] clusterConfig unchanged [2021-10-22T01:51:27.682+0000] [.error] [src/action/start.go:func1:93] [103] [01:51:27.682] Error starting mongod : [01:51:27.681] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39630,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39630 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:51:26.521Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 73556 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:51:26.543+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:51:26.543+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:51:26.543+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39630,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:51:26.545+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:51:26.547+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:51:26.568+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:51:26.587+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:51:26.587+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:51:26.587+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:51:26.633+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:51:26.633+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:51:27.179+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867487:179060][73556:0x7fed8b974bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:51:27.289+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867487:289286][73556:0x7fed8b974bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:51:27.389+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867487:389521][73556:0x7fed8b974bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:51:27.502+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867487:502587][73556:0x7fed8b974bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:51:27.578+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867487:578868][73556:0x7fed8b974bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:51:27.632+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867487:632837][73556:0x7fed8b974bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:51:27.632+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867487:632893][73556:0x7fed8b974bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:51:27.682+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:51:27.682] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39630 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:51:27.682] Error starting mongod : [01:51:27.681] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39630,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39630 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:51:26.521Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 73556 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:51:26.543+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:51:26.543+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:51:26.543+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39630,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:51:26.545+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:51:26.547+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:51:26.568+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:51:26.587+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:51:26.587+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:51:26.587+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:51:26.633+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:51:26.633+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:51:27.179+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867487:179060][73556:0x7fed8b974bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:51:27.289+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:51:27.682+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:51:27.682] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:51:27.682] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39630 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:51:27.682] Error starting mongod : [01:51:27.681] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39630,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39630 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:51:26.521Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 73556 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:51:26.543+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:51:26.543+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:51:26.543+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39630,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:51:26.545+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:51:26.547+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:51:26.568+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:51:26.587+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:51:26.587+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:51:26.587+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:51:26.633+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:51:26.633+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:51:27.179+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867487:179060][73556:0x7fed8b974bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:51:27.682+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:51:27.682] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:51:27.682+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:51:27.682] Initiating stopEphemeralMongo [2021-10-22T01:51:27.682+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:51:27.682] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:51:27.683+0000] [.error] [src/director/director.go:executePlan:962] [01:51:27.683] Failed to apply action. Result = : [01:51:27.682] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:51:27.682] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39630 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:51:27.682] Error starting mongod : [01:51:27.681] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39630,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39630 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:51:26.521Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 73556 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:51:26.543+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:51:26.543+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:51:26.543+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39630,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:51:26.545+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:51:26.547+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:51:26.568+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:51:26.587+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:51:26.587+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:51:26.587+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:51:26.633+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:51:26.633+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:51:27.179+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:51:27.683+0000] [.error] [src/director/director.go:planAndExecute:567] [01:51:27.683] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:51:27.683] Failed to apply action. Result = : [01:51:27.682] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:51:27.682] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39630 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:51:27.682] Error starting mongod : [01:51:27.681] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39630,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39630 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:51:26.521Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 73556 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:51:26.543+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:51:26.543+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:51:26.543+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39630,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:51:26.545+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:51:26.547+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:51:26.568+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:51:26.587+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:51:26.587+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:51:26.587+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:51:26.633+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:51:26.633+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:51:27.683+0000] [.error] [src/director/director.go:mainLoop:399] [01:51:27.683] Failed to planAndExecute : [01:51:27.683] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:51:27.683] Failed to apply action. Result = : [01:51:27.682] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:51:27.682] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39630 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:51:27.682] Error starting mongod : [01:51:27.681] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39630,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39630 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:51:26.521Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 73556 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:51:26.543+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:51:26.543+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:51:26.543+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39630,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:51:26.545+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:51:26.547+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:51:26.568+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:51:26.587+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:51:26.587+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:51:26.587+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:51:26.633+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:51:26.633+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:51:28.182+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:28.182] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:28.586+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:51:28.586] clusterConfig unchanged [2021-10-22T01:51:29.164+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:29.164] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:30.165+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:30.165] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:31.184+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:31.184] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:32.198+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:32.198] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:33.186+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:33.186] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:34.187+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:34.187] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:35.187+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:35.187] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:36.188+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:36.188] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:37.188+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:37.188] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:37.244+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:51:37.244] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 13.13453472s [2021-10-22T01:51:38.186+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:38.186] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:38.678+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:51:38.678] clusterConfig unchanged [2021-10-22T01:51:39.186+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:39.186] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:40.187+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:40.186] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:41.189+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:41.189] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:41.355+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:51:41.355] memory:rss=54263808 alloc=10634344 totalalloc=7803348968 mallocs=99467494 frees=99384239 heapinuse=16596992 heapobjects=83255 gcpausetotalns=1557297115 gcpauselastns=106614 gcnum=1187 utimens=156891523000 stimens=58528698000 minflt=408192 majflt=405 [2021-10-22T01:51:42.188+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:42.188] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:43.189+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:43.189] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:44.195+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:44.195] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:45.190+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:45.190] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:46.190+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:46.190] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:47.190+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:47.190] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:48.191+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:48.191] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:48.789+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:51:48.789] clusterConfig unchanged [2021-10-22T01:51:49.191+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:49.191] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:50.192+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:50.192] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:51.194+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:51.194] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:52.194+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:52.194] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:53.195+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:53.195] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:54.196+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:54.196] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:55.197+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:55.197] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:56.197+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:56.197] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:57.197+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:57.197] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:58.198+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:58.198] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:51:58.931+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:51:58.931] clusterConfig unchanged [2021-10-22T01:51:59.198+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:51:59.198] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:00.199+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:00.199] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:01.199+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:01.199] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:02.204+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:02.204] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:03.201+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:03.201] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:04.204+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:04.204] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:05.203+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:05.203] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:06.204+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:06.204] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:07.204+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:07.204] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:08.205+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:08.205] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:09.052+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:52:09.052] clusterConfig unchanged [2021-10-22T01:52:09.206+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:09.206] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:10.206+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:10.206] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:11.207+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:11.207] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:12.208+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:12.208] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:13.209+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:13.209] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:14.209+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:14.209] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:15.209+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:15.209] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:16.210+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:16.210] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:17.215+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:17.215] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:18.211+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:18.211] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:19.184+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:52:19.184] clusterConfig unchanged [2021-10-22T01:52:19.212+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:19.212] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:20.213+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:20.213] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:21.214+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:21.214] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:22.214+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:22.214] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:23.214+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:23.214] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:24.215+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:24.215] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:25.216+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:25.216] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:26.216+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:26.216] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:27.219+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:27.219] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:27.567+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:52:27.567] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:52:27.568+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:52:27.568] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:52:27.845+0000] [.info] [src/director/director.go:computePlan:280] [01:52:27.845] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:52:27.845+0000] [.info] [src/director/director.go:tracef:794] [01:52:27.845] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:52:27.845+0000] [.info] [src/director/director.go:tracef:794] [01:52:27.845] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:52:27.846+0000] [.info] [src/director/director.go:tracef:794] [01:52:27.846] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:52:27.846+0000] [.info] [src/director/director.go:tracef:794] [01:52:27.846] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:52:27.846+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:52:27.846] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:52:28.000+0000] [.info] [src/director/director.go:tracef:794] [01:52:28.000] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:52:28.000+0000] [.info] [src/director/director.go:tracef:794] [01:52:28.000] because [2021-10-22T01:52:28.000+0000] [.info] [src/director/director.go:tracef:794] [01:52:28.000] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:52:28.000+0000] [.info] [src/director/director.go:tracef:794] [01:52:28.000] because ['currentState.CanBounceStop' = true] [2021-10-22T01:52:28.176+0000] [.info] [src/director/director.go:tracef:794] [01:52:28.176] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:52:28.176+0000] [.info] [src/director/director.go:tracef:794] [01:52:28.176] because [2021-10-22T01:52:28.176+0000] [.info] [src/director/director.go:tracef:794] [01:52:28.176] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:52:28.176+0000] [.info] [src/director/director.go:tracef:794] [01:52:28.176] because ['currentState.Up' = false] [2021-10-22T01:52:28.176+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:52:28.176] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:52:28.176+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:52:28.176] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:52:28.217+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:28.217] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:28.266+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:52:28.266] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:52:28.274+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:52:28.274] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:52:28.274+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:52:28.274] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:52:28.275+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:52:28.275] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:52:28.276+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:52:28.276] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:52:28.276+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:52:28.276] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:52:28.276+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:52:28.276] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:52:28.278+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:52:28.278] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:52:28.488+0000] [.info] [src/director/director.go:tracef:794] [01:52:28.488] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:52:28.489+0000] [.info] [src/director/director.go:tracef:794] [01:52:28.489] because ['currentState.Up' = false] [2021-10-22T01:52:28.697+0000] [.info] [src/director/director.go:tracef:794] [01:52:28.697] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:52:28.697+0000] [.info] [src/director/director.go:tracef:794] [01:52:28.697] because ['currentState.Up' = false] [2021-10-22T01:52:28.697+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:52:28.697] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:52:28.909+0000] [.info] [src/director/director.go:tracef:794] [01:52:28.909] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:52:28.910+0000] [.info] [src/director/director.go:tracef:794] [01:52:28.910] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:52:29.130+0000] [.info] [src/director/director.go:tracef:794] [01:52:29.130] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:52:29.130+0000] [.info] [src/director/director.go:tracef:794] [01:52:29.130] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:52:29.130+0000] [.info] [src/director/director.go:tracef:794] [01:52:29.130] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:52:29.130+0000] [.info] [src/director/director.go:tracef:794] [01:52:29.130] because ['currentState.Up' = false] [2021-10-22T01:52:29.130+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:52:29.130] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:52:29.130+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:52:29.130] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:52:29.163+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:52:29.163] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:52:29.163+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:52:29.163] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38491 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:52:29.221+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:29.221] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:29.366+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:52:29.366] clusterConfig unchanged [2021-10-22T01:52:29.885+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:52:29.885] ...Finished start [pid=18750] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38491,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:52:30.036+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:52:30.036] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 151.272085ms. [2021-10-22T01:52:30.036+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:52:30.036] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38491 (local=false) [2021-10-22T01:52:30.036+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:52:30.036] Dropping existing documents from system.replset [2021-10-22T01:52:30.159+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:52:30.159] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:52:30.166+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:52:30.166] Inserting oplogTruncateAfterPoint document [2021-10-22T01:52:30.166+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:52:30.166] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:52:30.167+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:52:30.167] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:52:30.167+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:52:30.167] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:52:30.181+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:52:30.181] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:52:30.181+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:52:30.181] Setting the minvalid document to [{_id ObjectID("6172195e523370002becc6b9")} {t -1} {ts {0 1}}] [2021-10-22T01:52:30.195+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:52:30.195] Inserted [{_id ObjectID("6172195e523370002becc6b9")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:52:30.195+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:52:30.195] Inserting new replica set configuration document. [2021-10-22T01:52:30.195+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:52:30.195] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:52:30.208+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:52:30.208] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:52:30.211+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:52:30.211] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:52:30.211+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:52:30.211] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:52:30.211+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:52:30.211] Initiating stopEphemeralMongo [2021-10-22T01:52:30.211+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:52:30.211] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38491 (local=false) [2021-10-22T01:52:30.211+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:52:30.211] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38491 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:52:30.233+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:30.233] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:30.587+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:52:30.587] clusterConfig unchanged [2021-10-22T01:52:30.611+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:52:30.611] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.767721ms. [2021-10-22T01:52:30.611+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:52:30.611] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38491 (local=false) [2021-10-22T01:52:30.611+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:52:30.611] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 30.545µs. [2021-10-22T01:52:30.611+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:52:30.611] Successfully finished running stopEphemeralMongo [2021-10-22T01:52:30.611+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:52:30.611] Successfully stopped existing ephemeral mongod. [2021-10-22T01:52:30.611+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:52:30.611] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:52:30.611+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:52:30.611] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:52:30.646+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:52:30.646] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:52:30.646+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:52:30.646] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52949 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:52:31.235+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:31.235] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:31.719+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:52:31.719] clusterConfig unchanged [2021-10-22T01:52:31.836+0000] [.error] [src/action/start.go:func1:93] [103] [01:52:31.836] Error starting mongod : [01:52:31.836] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52949,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52949 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:52:30.678Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 20285 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:52:30.698+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:52:30.698+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:52:30.698+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52949,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:52:30.699+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:52:30.702+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:52:30.724+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:52:30.743+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:52:30.743+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:52:30.754+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:52:30.799+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:52:30.799+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:52:31.308+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867551:308647][20285:0x7f41a5983bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:52:31.412+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867551:412300][20285:0x7f41a5983bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:52:31.526+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867551:526693][20285:0x7f41a5983bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:52:31.639+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867551:639935][20285:0x7f41a5983bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:52:31.723+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867551:723424][20285:0x7f41a5983bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:52:31.786+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867551:786947][20285:0x7f41a5983bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:52:31.787+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867551:787027][20285:0x7f41a5983bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:52:31.836+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:52:31.836] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:52949 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:52:31.836] Error starting mongod : [01:52:31.836] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52949,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52949 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:52:30.678Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 20285 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:52:30.698+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:52:30.698+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:52:30.698+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52949,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:52:30.699+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:52:30.702+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:52:30.724+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:52:30.743+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:52:30.743+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:52:30.754+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:52:30.799+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:52:30.799+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:52:31.308+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867551:308647][20285:0x7f41a5983bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:52:31.412+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:52:31.837+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:52:31.837] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:52:31.836] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:52949 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:52:31.836] Error starting mongod : [01:52:31.836] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52949,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52949 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:52:30.678Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 20285 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:52:30.698+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:52:30.698+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:52:30.698+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52949,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:52:30.699+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:52:30.702+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:52:30.724+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:52:30.743+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:52:30.743+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:52:30.754+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:52:30.799+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:52:30.799+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:52:31.308+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867551:308647][20285:0x7f41a5983bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:52:31.837+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:52:31.837] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:52:31.837+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:52:31.837] Initiating stopEphemeralMongo [2021-10-22T01:52:31.837+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:52:31.837] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:52:31.837+0000] [.error] [src/director/director.go:executePlan:962] [01:52:31.837] Failed to apply action. Result = : [01:52:31.837] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:52:31.836] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:52949 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:52:31.836] Error starting mongod : [01:52:31.836] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52949,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52949 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:52:30.678Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 20285 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:52:30.698+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:52:30.698+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:52:30.698+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52949,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:52:30.699+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:52:30.702+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:52:30.724+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:52:30.743+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:52:30.743+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:52:30.754+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:52:30.799+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:52:30.799+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:52:31.308+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:52:31.837+0000] [.error] [src/director/director.go:planAndExecute:567] [01:52:31.837] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:52:31.837] Failed to apply action. Result = : [01:52:31.837] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:52:31.836] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:52949 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:52:31.836] Error starting mongod : [01:52:31.836] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52949,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52949 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:52:30.678Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 20285 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:52:30.698+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:52:30.698+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:52:30.698+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52949,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:52:30.699+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:52:30.702+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:52:30.724+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:52:30.743+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:52:30.743+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:52:30.754+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:52:30.799+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:52:30.799+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:52:31.838+0000] [.error] [src/director/director.go:mainLoop:399] [01:52:31.838] Failed to planAndExecute : [01:52:31.837] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:52:31.837] Failed to apply action. Result = : [01:52:31.837] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:52:31.836] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:52949 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:52:31.836] Error starting mongod : [01:52:31.836] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52949,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 52949 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:52:30.678Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 20285 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:52:30.698+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:52:30.698+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:52:30.698+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":52949,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:52:30.699+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:52:30.702+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:52:30.724+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:52:30.743+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:52:30.743+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:52:30.754+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:52:30.799+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:52:30.799+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:52:32.237+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:32.237] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:33.039+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:52:33.039] clusterConfig unchanged [2021-10-22T01:52:33.237+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:33.237] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:34.237+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:34.237] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:35.239+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:35.239] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:36.236+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:36.236] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:37.244+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [01:52:37.244] lsblk cache invalidated, running lsblk command [2021-10-22T01:52:37.246+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:37.246] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:38.253+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:38.253] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:39.251+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:39.251] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:40.267+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:40.267] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:40.906+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:52:40.906] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.627584706s [2021-10-22T01:52:41.252+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:41.252] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:41.356+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:52:41.356] memory:rss=56004608 alloc=8978264 totalalloc=7893385992 mallocs=100619250 frees=100555642 heapinuse=16367616 heapobjects=63608 gcpausetotalns=1558532099 gcpauselastns=126022 gcnum=1200 utimens=159081746000 stimens=59423903000 minflt=412958 majflt=405 [2021-10-22T01:52:42.251+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:42.251] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:43.170+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:52:43.170] clusterConfig unchanged [2021-10-22T01:52:43.251+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:43.251] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:44.252+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:44.252] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:45.253+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:45.253] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:46.253+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:46.253] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:47.254+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:47.254] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:48.254+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:48.254] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:49.255+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:49.255] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:50.256+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:50.256] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:51.263+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:51.263] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:52.256+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:52.255] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:53.256+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:53.256] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:53.274+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:52:53.274] clusterConfig unchanged [2021-10-22T01:52:54.256+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:54.256] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:55.257+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:55.257] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:56.257+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:56.257] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:57.263+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:57.263] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:58.258+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:58.258] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:52:59.259+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:52:59.259] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:00.260+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:00.260] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:01.260+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:01.260] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:02.263+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:02.263] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:03.262+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:03.262] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:03.395+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:53:03.395] clusterConfig unchanged [2021-10-22T01:53:04.262+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:04.262] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:05.264+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:05.264] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:06.265+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:06.264] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:07.265+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:07.265] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:08.265+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:08.265] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:09.265+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:09.265] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:10.266+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:10.266] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:11.267+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:11.267] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:12.269+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:12.269] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:13.268+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:13.268] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:13.584+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:53:13.584] clusterConfig unchanged [2021-10-22T01:53:14.269+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:14.269] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:15.270+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:15.270] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:16.270+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:16.270] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:17.271+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:17.271] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:18.271+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:18.271] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:19.273+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:19.273] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:20.272+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:20.272] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:21.273+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:21.273] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:22.273+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:22.273] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:23.275+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:23.275] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:23.726+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:53:23.726] clusterConfig unchanged [2021-10-22T01:53:24.274+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:24.274] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:25.276+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:25.276] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:26.275+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:26.275] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:27.276+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:27.276] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:28.277+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:28.277] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:29.277+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:29.277] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:30.277+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:30.277] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:31.282+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:31.282] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:32.003+0000] [.info] [src/director/director.go:computePlan:280] [01:53:32.003] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:53:32.003+0000] [.info] [src/director/director.go:tracef:794] [01:53:32.003] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:53:32.003+0000] [.info] [src/director/director.go:tracef:794] [01:53:32.003] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:53:32.003+0000] [.info] [src/director/director.go:tracef:794] [01:53:32.003] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:53:32.003+0000] [.info] [src/director/director.go:tracef:794] [01:53:32.003] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:53:32.003+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:53:32.003] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:53:32.160+0000] [.info] [src/director/director.go:tracef:794] [01:53:32.160] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:53:32.160+0000] [.info] [src/director/director.go:tracef:794] [01:53:32.160] because [2021-10-22T01:53:32.160+0000] [.info] [src/director/director.go:tracef:794] [01:53:32.160] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:53:32.160+0000] [.info] [src/director/director.go:tracef:794] [01:53:32.160] because ['currentState.CanBounceStop' = true] [2021-10-22T01:53:32.281+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:32.281] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:32.331+0000] [.info] [src/director/director.go:tracef:794] [01:53:32.331] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:53:32.331+0000] [.info] [src/director/director.go:tracef:794] [01:53:32.331] because [2021-10-22T01:53:32.332+0000] [.info] [src/director/director.go:tracef:794] [01:53:32.332] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:53:32.332+0000] [.info] [src/director/director.go:tracef:794] [01:53:32.332] because ['currentState.Up' = false] [2021-10-22T01:53:32.332+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:53:32.332] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:53:32.332+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:53:32.332] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:53:32.519+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:53:32.519] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:53:32.527+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:53:32.527] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:53:32.528+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:53:32.528] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:53:32.528+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:53:32.528] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:53:32.529+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:53:32.529] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:53:32.529+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:53:32.529] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:53:32.529+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:53:32.529] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:53:32.531+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:53:32.531] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:53:32.747+0000] [.info] [src/director/director.go:tracef:794] [01:53:32.747] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:53:32.747+0000] [.info] [src/director/director.go:tracef:794] [01:53:32.747] because ['currentState.Up' = false] [2021-10-22T01:53:32.959+0000] [.info] [src/director/director.go:tracef:794] [01:53:32.959] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:53:32.959+0000] [.info] [src/director/director.go:tracef:794] [01:53:32.959] because ['currentState.Up' = false] [2021-10-22T01:53:32.959+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:53:32.959] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:53:33.165+0000] [.info] [src/director/director.go:tracef:794] [01:53:33.165] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:53:33.165+0000] [.info] [src/director/director.go:tracef:794] [01:53:33.165] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:53:33.281+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:33.281] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:33.388+0000] [.info] [src/director/director.go:tracef:794] [01:53:33.388] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:53:33.388+0000] [.info] [src/director/director.go:tracef:794] [01:53:33.388] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:53:33.389+0000] [.info] [src/director/director.go:tracef:794] [01:53:33.389] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:53:33.389+0000] [.info] [src/director/director.go:tracef:794] [01:53:33.389] because ['currentState.Up' = false] [2021-10-22T01:53:33.389+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:53:33.389] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:53:33.389+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:53:33.389] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:53:33.423+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:53:33.423] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:53:33.423+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:53:33.423] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50500 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:53:33.917+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:53:33.917] clusterConfig unchanged [2021-10-22T01:53:34.137+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:53:34.137] ...Finished start [pid=39554] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50500,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:53:34.253+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:53:34.253] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 115.206613ms. [2021-10-22T01:53:34.253+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:53:34.253] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50500 (local=false) [2021-10-22T01:53:34.253+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:53:34.253] Dropping existing documents from system.replset [2021-10-22T01:53:34.293+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:34.293] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:34.370+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:53:34.370] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:53:34.374+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:53:34.374] Inserting oplogTruncateAfterPoint document [2021-10-22T01:53:34.374+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:53:34.374] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:53:34.375+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:53:34.375] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:53:34.375+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:53:34.375] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:53:34.388+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:53:34.388] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:53:34.388+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:53:34.388] Setting the minvalid document to [{_id ObjectID("6172199e523370002becc6c7")} {t -1} {ts {0 1}}] [2021-10-22T01:53:34.401+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:53:34.401] Inserted [{_id ObjectID("6172199e523370002becc6c7")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:53:34.401+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:53:34.401] Inserting new replica set configuration document. [2021-10-22T01:53:34.401+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:53:34.401] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:53:34.413+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:53:34.413] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:53:34.418+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:53:34.418] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:53:34.418+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:53:34.418] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:53:34.418+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:53:34.418] Initiating stopEphemeralMongo [2021-10-22T01:53:34.418+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:53:34.418] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50500 (local=false) [2021-10-22T01:53:34.418+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:53:34.418] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50500 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:53:34.761+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:53:34.761] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.546553ms. [2021-10-22T01:53:34.761+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:53:34.761] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50500 (local=false) [2021-10-22T01:53:34.761+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:53:34.761] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 40.114µs. [2021-10-22T01:53:34.761+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:53:34.761] Successfully finished running stopEphemeralMongo [2021-10-22T01:53:34.761+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:53:34.761] Successfully stopped existing ephemeral mongod. [2021-10-22T01:53:34.761+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:53:34.761] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:53:34.762+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:53:34.761] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:53:34.793+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:53:34.793] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:53:34.794+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:53:34.794] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49911 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:53:35.049+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:53:35.049] clusterConfig unchanged [2021-10-22T01:53:35.298+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:35.298] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:36.020+0000] [.error] [src/action/start.go:func1:93] [103] [01:53:36.020] Error starting mongod : [01:53:36.020] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49911 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 40922 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:53:34.843+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:53:34.843+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:53:34.843+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:53:34.845+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:53:34.847+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:53:34.848+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:53:34.860+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:53:34.860+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:53:34.860+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:53:34.905+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:53:34.905+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:53:35.464+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867615:464715][40922:0x7f29a80f2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:53:35.592+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867615:592836][40922:0x7f29a80f2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:53:35.708+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867615:708842][40922:0x7f29a80f2bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/33920 to 2/256"}} {"t":{"$date":"2021-10-22T01:53:35.836+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867615:836544][40922:0x7f29a80f2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:53:35.909+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867615:909823][40922:0x7f29a80f2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:53:35.969+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867615:969217][40922:0x7f29a80f2bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:53:35.969+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867615:969292][40922:0x7f29a80f2bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:53:35.970+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:53:36.020+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:53:36.020] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49911 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:53:36.020] Error starting mongod : [01:53:36.020] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49911 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 40922 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:53:34.843+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:53:34.843+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:53:34.843+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:53:34.845+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:53:34.847+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:53:34.848+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:53:34.860+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:53:34.860+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:53:34.860+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:53:34.905+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:53:34.905+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:53:35.464+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867615:464715][40922:0x7f29a80f2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:53:35.592+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867615:592836][40922:0x7f29a80f2bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:53:36.020+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:53:36.020] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:53:36.020] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49911 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:53:36.020] Error starting mongod : [01:53:36.020] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49911 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 40922 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:53:34.843+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:53:34.843+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:53:34.843+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:53:34.845+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:53:34.847+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:53:34.848+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:53:34.860+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:53:34.860+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:53:34.860+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:53:34.905+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:53:34.905+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:53:35.464+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867615:464715][40922:0x7f29a80f2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:53:35.592+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:53:36.021+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:53:36.021] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:53:36.021+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:53:36.021] Initiating stopEphemeralMongo [2021-10-22T01:53:36.021+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:53:36.021] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:53:36.021+0000] [.error] [src/director/director.go:executePlan:962] [01:53:36.021] Failed to apply action. Result = : [01:53:36.020] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:53:36.020] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49911 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:53:36.020] Error starting mongod : [01:53:36.020] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49911 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 40922 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:53:34.843+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:53:34.843+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:53:34.843+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:53:34.845+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:53:34.847+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:53:34.848+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:53:34.860+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:53:34.860+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:53:34.860+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:53:34.905+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:53:34.905+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:53:35.464+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867615:464715][40922:0x7f29a80f2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:53:36.021+0000] [.error] [src/director/director.go:planAndExecute:567] [01:53:36.021] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:53:36.021] Failed to apply action. Result = : [01:53:36.020] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:53:36.020] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49911 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:53:36.020] Error starting mongod : [01:53:36.020] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49911 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 40922 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:53:34.843+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:53:34.843+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:53:34.843+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:53:34.845+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:53:34.847+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:53:34.848+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:53:34.860+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:53:34.860+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:53:34.860+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:53:34.905+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:53:34.905+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:53:35.464+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:53:36.021+0000] [.error] [src/director/director.go:mainLoop:399] [01:53:36.021] Failed to planAndExecute : [01:53:36.021] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:53:36.021] Failed to apply action. Result = : [01:53:36.020] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:53:36.020] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49911 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:53:36.020] Error starting mongod : [01:53:36.020] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49911 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 40922 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:53:34.843+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:53:34.843+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:53:34.843+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49911,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:53:34.845+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:53:34.847+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:53:34.848+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:53:34.860+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:53:34.860+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:53:34.860+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:53:34.905+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:53:34.905+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:53:35.464+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:53:36.298+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:36.298] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:36.310+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:53:36.310] clusterConfig unchanged [2021-10-22T01:53:37.300+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:37.300] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:38.299+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:38.299] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:39.301+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:39.301] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:40.304+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:40.304] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:41.300+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:41.300] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:41.367+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:53:41.367] memory:rss=56401920 alloc=11929264 totalalloc=7982514272 mallocs=101762586 frees=101670830 heapinuse=17481728 heapobjects=91756 gcpausetotalns=1560476189 gcpauselastns=116249 gcnum=1212 utimens=161069403000 stimens=60208364000 minflt=418125 majflt=405 [2021-10-22T01:53:42.306+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:42.306] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:43.304+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:43.303] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:44.312+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:44.312] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:44.340+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [01:53:44.340] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49911 (local=false) because it is not up [2021-10-22T01:53:44.935+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:53:44.935] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.404602016s [2021-10-22T01:53:45.309+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:45.309] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:46.309+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:46.309] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:46.411+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:53:46.411] clusterConfig unchanged [2021-10-22T01:53:47.310+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:47.310] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:48.311+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:48.311] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:49.311+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:49.311] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:50.312+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:50.312] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:51.313+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:51.313] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:52.313+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:52.313] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:53.314+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:53.314] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:54.314+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:54.314] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:55.314+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:55.314] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:56.315+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:56.315] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:56.504+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:53:56.504] clusterConfig unchanged [2021-10-22T01:53:57.315+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:57.315] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:58.316+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:58.316] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:53:59.317+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:53:59.317] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:00.317+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:00.317] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:01.318+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:01.318] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:02.321+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:02.321] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:03.318+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:03.318] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:04.319+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:04.319] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:05.319+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:05.319] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:06.319+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:06.319] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:06.635+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:54:06.635] clusterConfig unchanged [2021-10-22T01:54:07.322+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:07.322] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:08.320+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:08.320] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:09.321+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:09.321] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:10.322+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:10.322] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:11.324+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:11.324] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:12.324+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:12.324] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:13.324+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:13.324] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:14.324+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:14.324] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:15.325+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:15.325] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:16.326+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:16.326] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:16.757+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:54:16.757] clusterConfig unchanged [2021-10-22T01:54:17.326+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:17.326] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:18.335+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:18.335] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:19.328+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:19.328] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:20.329+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:20.329] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:21.330+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:21.330] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:22.329+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:22.329] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:23.330+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:23.329] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:24.330+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:24.330] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:25.330+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:25.330] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:26.331+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:26.331] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:26.899+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:54:26.899] clusterConfig unchanged [2021-10-22T01:54:27.331+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:27.331] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:28.331+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:28.331] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:29.339+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:29.339] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:30.333+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:30.333] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:31.334+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:31.334] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:32.335+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:32.335] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:33.336+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:33.336] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:34.336+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:34.336] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:35.336+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:35.336] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:36.196+0000] [.info] [src/director/director.go:computePlan:280] [01:54:36.196] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:54:36.196+0000] [.info] [src/director/director.go:tracef:794] [01:54:36.196] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:54:36.196+0000] [.info] [src/director/director.go:tracef:794] [01:54:36.196] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:54:36.196+0000] [.info] [src/director/director.go:tracef:794] [01:54:36.196] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:54:36.196+0000] [.info] [src/director/director.go:tracef:794] [01:54:36.196] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:54:36.196+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:54:36.196] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:54:36.341+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:36.341] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:36.364+0000] [.info] [src/director/director.go:tracef:794] [01:54:36.364] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:54:36.364+0000] [.info] [src/director/director.go:tracef:794] [01:54:36.364] because [2021-10-22T01:54:36.364+0000] [.info] [src/director/director.go:tracef:794] [01:54:36.364] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:54:36.364+0000] [.info] [src/director/director.go:tracef:794] [01:54:36.364] because ['currentState.CanBounceStop' = true] [2021-10-22T01:54:36.527+0000] [.info] [src/director/director.go:tracef:794] [01:54:36.527] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:54:36.527+0000] [.info] [src/director/director.go:tracef:794] [01:54:36.527] because [2021-10-22T01:54:36.527+0000] [.info] [src/director/director.go:tracef:794] [01:54:36.527] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:54:36.527+0000] [.info] [src/director/director.go:tracef:794] [01:54:36.527] because ['currentState.Up' = false] [2021-10-22T01:54:36.527+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:54:36.527] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:54:36.527+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:54:36.527] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:54:36.607+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:54:36.607] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:54:36.619+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:54:36.619] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:54:36.620+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:54:36.620] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:54:36.622+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:54:36.622] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:54:36.622+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:54:36.622] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:54:36.622+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:54:36.622] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:54:36.624+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:54:36.624] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:54:36.820+0000] [.info] [src/director/director.go:tracef:794] [01:54:36.820] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:54:36.820+0000] [.info] [src/director/director.go:tracef:794] [01:54:36.820] because ['currentState.Up' = false] [2021-10-22T01:54:37.043+0000] [.info] [src/director/director.go:tracef:794] [01:54:37.043] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:54:37.043+0000] [.info] [src/director/director.go:tracef:794] [01:54:37.043] because ['currentState.Up' = false] [2021-10-22T01:54:37.043+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:54:37.043] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:54:37.090+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:54:37.090] clusterConfig unchanged [2021-10-22T01:54:37.267+0000] [.info] [src/director/director.go:tracef:794] [01:54:37.267] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:54:37.267+0000] [.info] [src/director/director.go:tracef:794] [01:54:37.267] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:54:37.356+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:37.356] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:37.495+0000] [.info] [src/director/director.go:tracef:794] [01:54:37.495] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:54:37.495+0000] [.info] [src/director/director.go:tracef:794] [01:54:37.495] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:54:37.495+0000] [.info] [src/director/director.go:tracef:794] [01:54:37.495] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:54:37.495+0000] [.info] [src/director/director.go:tracef:794] [01:54:37.495] because ['currentState.Up' = false] [2021-10-22T01:54:37.495+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:54:37.495] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:54:37.495+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:54:37.495] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:54:37.531+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:54:37.531] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:54:37.531+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:54:37.531] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36502 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:54:38.212+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:54:38.212] clusterConfig unchanged [2021-10-22T01:54:38.272+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:54:38.272] ...Finished start [pid=60349] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36502,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:54:38.343+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:38.343] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:38.431+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:54:38.431] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 158.591994ms. [2021-10-22T01:54:38.431+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:54:38.431] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36502 (local=false) [2021-10-22T01:54:38.431+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:54:38.431] Dropping existing documents from system.replset [2021-10-22T01:54:38.633+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:54:38.633] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:54:38.636+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:54:38.636] Inserting oplogTruncateAfterPoint document [2021-10-22T01:54:38.636+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:54:38.636] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:54:38.637+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:54:38.637] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:54:38.637+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:54:38.637] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:54:38.650+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:54:38.650] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:54:38.650+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:54:38.650] Setting the minvalid document to [{_id ObjectID("617219de523370002becc6d5")} {t -1} {ts {0 1}}] [2021-10-22T01:54:38.662+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:54:38.662] Inserted [{_id ObjectID("617219de523370002becc6d5")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:54:38.662+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:54:38.662] Inserting new replica set configuration document. [2021-10-22T01:54:38.662+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:54:38.662] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:54:38.674+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:54:38.674] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:54:38.682+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:54:38.682] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:54:38.682+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:54:38.682] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:54:38.682+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:54:38.682] Initiating stopEphemeralMongo [2021-10-22T01:54:38.682+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:54:38.682] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36502 (local=false) [2021-10-22T01:54:38.682+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:54:38.682] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36502 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:54:39.156+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:54:39.156] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.223512ms. [2021-10-22T01:54:39.156+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:54:39.156] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36502 (local=false) [2021-10-22T01:54:39.156+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:54:39.156] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.243µs. [2021-10-22T01:54:39.156+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:54:39.156] Successfully finished running stopEphemeralMongo [2021-10-22T01:54:39.156+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:54:39.156] Successfully stopped existing ephemeral mongod. [2021-10-22T01:54:39.156+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:54:39.156] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:54:39.156+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:54:39.156] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:54:39.191+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:54:39.191] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:54:39.191+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:54:39.191] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50622 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:54:39.342+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:39.342] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:39.395+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:54:39.395] clusterConfig unchanged [2021-10-22T01:54:40.344+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:40.344] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:40.496+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:54:40.496] clusterConfig unchanged [2021-10-22T01:54:40.500+0000] [.error] [src/action/start.go:func1:93] [103] [01:54:40.500] Error starting mongod : [01:54:40.500] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50622,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50622 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:54:39.224Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 62107 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:54:39.242+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:54:39.242+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:54:39.242+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50622,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:54:39.245+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:54:39.247+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:54:39.258+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:54:39.259+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:54:39.259+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:54:39.260+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:54:39.305+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:54:39.305+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:54:39.885+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867679:885088][62107:0x7f7895922bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:54:40.054+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867680:54108][62107:0x7f7895922bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:54:40.179+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867680:179327][62107:0x7f7895922bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:54:40.307+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867680:307513][62107:0x7f7895922bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:54:40.380+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867680:380268][62107:0x7f7895922bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:54:40.441+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867680:441194][62107:0x7f7895922bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:54:40.441+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867680:441269][62107:0x7f7895922bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:54:40.500+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:54:40.500] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:50622 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:54:40.500] Error starting mongod : [01:54:40.500] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50622,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50622 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:54:39.224Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 62107 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:54:39.242+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:54:39.242+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:54:39.242+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50622,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:54:39.245+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:54:39.247+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:54:39.258+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:54:39.259+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:54:39.259+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:54:39.260+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:54:39.305+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:54:39.305+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:54:39.885+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867679:885088][62107:0x7f7895922bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:54:40.054+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:54:40.500+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:54:40.500] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:54:40.500] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:50622 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:54:40.500] Error starting mongod : [01:54:40.500] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50622,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50622 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:54:39.224Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 62107 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:54:39.242+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:54:39.242+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:54:39.242+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50622,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:54:39.245+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:54:39.247+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:54:39.258+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:54:39.259+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:54:39.259+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:54:39.260+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:54:39.305+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:54:39.305+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:54:39.885+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867679:885088][62107:0x7f7895922bc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:54:40.500+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:54:40.500] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:54:40.500+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:54:40.500] Initiating stopEphemeralMongo [2021-10-22T01:54:40.500+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:54:40.500] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:54:40.501+0000] [.error] [src/director/director.go:executePlan:962] [01:54:40.501] Failed to apply action. Result = : [01:54:40.500] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:54:40.500] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:50622 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:54:40.500] Error starting mongod : [01:54:40.500] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50622,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50622 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:54:39.224Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 62107 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:54:39.242+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:54:39.242+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:54:39.242+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50622,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:54:39.245+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:54:39.247+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:54:39.258+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:54:39.259+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:54:39.259+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:54:39.260+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:54:39.305+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:54:39.305+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:54:39.885+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:54:40.501+0000] [.error] [src/director/director.go:planAndExecute:567] [01:54:40.501] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:54:40.501] Failed to apply action. Result = : [01:54:40.500] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:54:40.500] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:50622 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:54:40.500] Error starting mongod : [01:54:40.500] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50622,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50622 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:54:39.224Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 62107 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:54:39.242+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:54:39.242+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:54:39.242+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50622,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:54:39.245+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:54:39.247+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:54:39.258+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:54:39.259+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:54:39.259+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:54:39.260+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:54:39.305+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:54:39.305+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:54:40.501+0000] [.error] [src/director/director.go:mainLoop:399] [01:54:40.501] Failed to planAndExecute : [01:54:40.501] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:54:40.501] Failed to apply action. Result = : [01:54:40.500] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:54:40.500] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:50622 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:54:40.500] Error starting mongod : [01:54:40.500] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50622,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50622 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:54:39.224Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 62107 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:54:39.242+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:54:39.242+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:54:39.242+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50622,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:54:39.245+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:54:39.247+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:54:39.258+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:54:39.259+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:54:39.259+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:54:39.260+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:54:39.305+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:54:39.305+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:54:41.343+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:41.343] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:41.368+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:54:41.368] memory:rss=55885824 alloc=14452608 totalalloc=8071526976 mallocs=102906112 frees=102800023 heapinuse=18890752 heapobjects=106089 gcpausetotalns=1561837208 gcpauselastns=82224 gcnum=1224 utimens=163071501000 stimens=61051065000 minflt=423536 majflt=405 [2021-10-22T01:54:41.727+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:54:41.727] clusterConfig unchanged [2021-10-22T01:54:42.343+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:42.343] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:43.343+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:43.343] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:44.344+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:44.344] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:45.345+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:45.345] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:46.361+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:46.361] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:47.345+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:47.345] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:48.345+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:48.345] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:49.251+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:54:49.251] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.627360639s [2021-10-22T01:54:49.345+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:49.345] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:50.345+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:50.345] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:51.347+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:51.347] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:51.849+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:54:51.849] clusterConfig unchanged [2021-10-22T01:54:52.354+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:52.354] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:53.348+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:53.348] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:54.348+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:54.348] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:55.349+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:55.349] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:56.349+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:56.349] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:57.350+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:57.350] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:58.350+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:58.350] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:54:59.351+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:54:59.351] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:00.352+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:00.352] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:01.354+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:01.354] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:01.980+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:55:01.980] clusterConfig unchanged [2021-10-22T01:55:02.353+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:02.353] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:03.353+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:03.353] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:04.354+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:04.354] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:05.355+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:05.355] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:06.355+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:06.355] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:07.355+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:07.355] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:08.356+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:08.356] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:09.357+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:09.357] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:10.358+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:10.358] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:11.358+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:11.358] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:12.092+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:55:12.092] clusterConfig unchanged [2021-10-22T01:55:12.359+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:12.359] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:13.360+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:13.360] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:14.361+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:14.361] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:15.362+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:15.362] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:16.362+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:16.362] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:17.363+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:17.363] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:18.364+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:18.364] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:19.366+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:19.366] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:20.366+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:20.366] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:21.367+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:21.367] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:22.195+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:55:22.195] clusterConfig unchanged [2021-10-22T01:55:22.367+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:22.367] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:23.368+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:23.368] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:24.369+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:24.369] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:25.376+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:25.376] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:26.370+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:26.370] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:27.371+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:27.371] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:28.372+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:28.372] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:29.372+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:29.372] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:30.373+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:30.373] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:31.372+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:31.372] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:32.327+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:55:32.327] clusterConfig unchanged [2021-10-22T01:55:32.373+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:32.373] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:33.373+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:33.373] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:34.374+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:34.374] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:35.374+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:35.374] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:36.385+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:36.385] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:37.375+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:37.375] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:38.377+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:38.377] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:39.376+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:39.376] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:40.377+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:40.377] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:40.665+0000] [.info] [src/director/director.go:computePlan:280] [01:55:40.665] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:55:40.665+0000] [.info] [src/director/director.go:tracef:794] [01:55:40.665] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:55:40.665+0000] [.info] [src/director/director.go:tracef:794] [01:55:40.665] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:55:40.665+0000] [.info] [src/director/director.go:tracef:794] [01:55:40.665] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:55:40.665+0000] [.info] [src/director/director.go:tracef:794] [01:55:40.665] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:55:40.665+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:55:40.665] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:55:40.832+0000] [.info] [src/director/director.go:tracef:794] [01:55:40.832] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:55:40.832+0000] [.info] [src/director/director.go:tracef:794] [01:55:40.832] because [2021-10-22T01:55:40.833+0000] [.info] [src/director/director.go:tracef:794] [01:55:40.833] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:55:40.833+0000] [.info] [src/director/director.go:tracef:794] [01:55:40.833] because ['currentState.CanBounceStop' = true] [2021-10-22T01:55:40.987+0000] [.info] [src/director/director.go:tracef:794] [01:55:40.987] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:55:40.987+0000] [.info] [src/director/director.go:tracef:794] [01:55:40.987] because [2021-10-22T01:55:40.987+0000] [.info] [src/director/director.go:tracef:794] [01:55:40.987] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:55:40.987+0000] [.info] [src/director/director.go:tracef:794] [01:55:40.987] because ['currentState.Up' = false] [2021-10-22T01:55:40.987+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:55:40.987] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:55:40.987+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:55:40.987] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:55:41.070+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:55:41.070] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:55:41.079+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:55:41.079] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:55:41.079+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:55:41.079] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:55:41.080+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:55:41.080] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:55:41.081+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:55:41.081] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:55:41.081+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:55:41.081] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:55:41.081+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:55:41.081] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:55:41.082+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:55:41.082] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:55:41.313+0000] [.info] [src/director/director.go:tracef:794] [01:55:41.313] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:55:41.313+0000] [.info] [src/director/director.go:tracef:794] [01:55:41.313] because ['currentState.Up' = false] [2021-10-22T01:55:41.369+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:55:41.369] memory:rss=54910976 alloc=15057328 totalalloc=8136487904 mallocs=103757227 frees=103615216 heapinuse=18538496 heapobjects=142011 gcpausetotalns=1562743756 gcpauselastns=106011 gcnum=1233 utimens=164589204000 stimens=61810046000 minflt=428066 majflt=405 [2021-10-22T01:55:41.409+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:41.409] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:41.540+0000] [.info] [src/director/director.go:tracef:794] [01:55:41.540] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:55:41.540+0000] [.info] [src/director/director.go:tracef:794] [01:55:41.540] because ['currentState.Up' = false] [2021-10-22T01:55:41.541+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:55:41.540] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:55:41.739+0000] [.info] [src/director/director.go:tracef:794] [01:55:41.739] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:55:41.739+0000] [.info] [src/director/director.go:tracef:794] [01:55:41.739] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:55:41.958+0000] [.info] [src/director/director.go:tracef:794] [01:55:41.958] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:55:41.958+0000] [.info] [src/director/director.go:tracef:794] [01:55:41.958] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:55:41.958+0000] [.info] [src/director/director.go:tracef:794] [01:55:41.958] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:55:41.958+0000] [.info] [src/director/director.go:tracef:794] [01:55:41.958] because ['currentState.Up' = false] [2021-10-22T01:55:41.958+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:55:41.958] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:55:41.958+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:55:41.958] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:55:41.992+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:55:41.992] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:55:41.993+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:55:41.993] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41299 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:55:42.380+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:42.380] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:42.576+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:55:42.576] clusterConfig unchanged [2021-10-22T01:55:42.720+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:55:42.720] ...Finished start [pid=7073] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41299,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:55:42.876+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:55:42.876] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 155.549275ms. [2021-10-22T01:55:42.876+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:55:42.876] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41299 (local=false) [2021-10-22T01:55:42.876+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:55:42.876] Dropping existing documents from system.replset [2021-10-22T01:55:43.030+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:55:43.030] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:55:43.033+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:55:43.033] Inserting oplogTruncateAfterPoint document [2021-10-22T01:55:43.033+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:55:43.033] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:55:43.035+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:55:43.035] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:55:43.035+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:55:43.035] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:55:43.047+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:55:43.047] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:55:43.047+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:55:43.047] Setting the minvalid document to [{_id ObjectID("61721a1f523370002becc6e3")} {t -1} {ts {0 1}}] [2021-10-22T01:55:43.060+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:55:43.060] Inserted [{_id ObjectID("61721a1f523370002becc6e3")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:55:43.060+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:55:43.060] Inserting new replica set configuration document. [2021-10-22T01:55:43.060+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:55:43.060] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:55:43.074+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:55:43.074] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:55:43.087+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:55:43.087] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:55:43.087+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:55:43.087] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:55:43.087+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:55:43.087] Initiating stopEphemeralMongo [2021-10-22T01:55:43.087+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:55:43.087] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41299 (local=false) [2021-10-22T01:55:43.087+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:55:43.087] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41299 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:55:43.379+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:43.379] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:43.649+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:55:43.649] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.87265ms. [2021-10-22T01:55:43.649+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:55:43.649] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41299 (local=false) [2021-10-22T01:55:43.650+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:55:43.649] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 52.613µs. [2021-10-22T01:55:43.650+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:55:43.650] Successfully finished running stopEphemeralMongo [2021-10-22T01:55:43.650+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:55:43.650] Successfully stopped existing ephemeral mongod. [2021-10-22T01:55:43.650+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:55:43.650] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:55:43.650+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:55:43.650] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:55:43.685+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:55:43.685] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:55:43.686+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:55:43.686] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34751 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:55:43.698+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:55:43.698] clusterConfig unchanged [2021-10-22T01:55:44.386+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:44.386] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:44.798+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:55:44.798] clusterConfig unchanged [2021-10-22T01:55:44.886+0000] [.error] [src/action/start.go:func1:93] [103] [01:55:44.886] Error starting mongod : [01:55:44.886] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34751,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34751 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 8824 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:55:43.751+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:55:43.751+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:55:43.751+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34751,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:55:43.752+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:55:43.754+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:55:43.755+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:55:43.776+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:55:43.776+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:55:43.783+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:55:43.829+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:55:43.829+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:55:44.357+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867744:357924][8824:0x7feccb4e5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:55:44.471+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867744:471174][8824:0x7feccb4e5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:55:44.570+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867744:570392][8824:0x7feccb4e5bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T01:55:44.684+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867744:684001][8824:0x7feccb4e5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:55:44.765+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867744:765932][8824:0x7feccb4e5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:55:44.838+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867744:838559][8824:0x7feccb4e5bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:55:44.838+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867744:838628][8824:0x7feccb4e5bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:55:44.840+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:55:44.886+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:55:44.886] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34751 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:55:44.886] Error starting mongod : [01:55:44.886] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34751,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34751 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 8824 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:55:43.751+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:55:43.751+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:55:43.751+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34751,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:55:43.752+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:55:43.754+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:55:43.755+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:55:43.776+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:55:43.776+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:55:43.783+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:55:43.829+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:55:43.829+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:55:44.357+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867744:357924][8824:0x7feccb4e5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:55:44.471+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867744:471174][8824:0x7feccb4e5bc0], txn-recover...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:55:44.886+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:55:44.886] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:55:44.886] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34751 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:55:44.886] Error starting mongod : [01:55:44.886] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34751,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34751 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 8824 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:55:43.751+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:55:43.751+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:55:43.751+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34751,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:55:43.752+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:55:43.754+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:55:43.755+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:55:43.776+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:55:43.776+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:55:43.783+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:55:43.829+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:55:43.829+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:55:44.357+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867744:357924][8824:0x7feccb4e5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:55:44.471+00:00"},"s":"I", "c":"STORAGE", "id":22430, ... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:55:44.886+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:55:44.886] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:55:44.886+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:55:44.886] Initiating stopEphemeralMongo [2021-10-22T01:55:44.886+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:55:44.886] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:55:44.887+0000] [.error] [src/director/director.go:executePlan:962] [01:55:44.887] Failed to apply action. Result = : [01:55:44.886] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:55:44.886] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34751 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:55:44.886] Error starting mongod : [01:55:44.886] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34751,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34751 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 8824 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:55:43.751+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:55:43.751+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:55:43.751+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34751,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:55:43.752+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:55:43.754+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:55:43.755+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:55:43.776+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:55:43.776+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:55:43.783+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:55:43.829+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:55:43.829+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:55:44.357+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867744:357924][8824:0x7feccb4e5bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:55:44.887+0000] [.error] [src/director/director.go:planAndExecute:567] [01:55:44.887] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:55:44.887] Failed to apply action. Result = : [01:55:44.886] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:55:44.886] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34751 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:55:44.886] Error starting mongod : [01:55:44.886] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34751,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34751 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 8824 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:55:43.751+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:55:43.751+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:55:43.751+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34751,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:55:43.752+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:55:43.754+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:55:43.755+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:55:43.776+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:55:43.776+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:55:43.783+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:55:43.829+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:55:43.829+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:55:44.357+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger m... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:55:44.887+0000] [.error] [src/director/director.go:mainLoop:399] [01:55:44.887] Failed to planAndExecute : [01:55:44.887] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:55:44.887] Failed to apply action. Result = : [01:55:44.886] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:55:44.886] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34751 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:55:44.886] Error starting mongod : [01:55:44.886] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34751,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34751 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 8824 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:55:43.751+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:55:43.751+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:55:43.751+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34751,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:55:43.752+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:55:43.754+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:55:43.755+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:55:43.776+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:55:43.776+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:55:43.783+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:55:43.829+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:55:43.829+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:55:44.357+00:00"},"s":"I",... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:55:45.408+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:45.408] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:46.042+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:55:46.042] clusterConfig unchanged [2021-10-22T01:55:46.394+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:46.394] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:47.392+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:47.392] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:48.395+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:48.395] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:49.393+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:49.393] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:50.396+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:50.396] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:51.397+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:51.397] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:52.398+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:52.398] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:53.414+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:53.414] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:53.722+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:55:53.722] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.639746686s [2021-10-22T01:55:54.396+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:54.396] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:55.397+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:55.397] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:56.166+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:55:56.166] clusterConfig unchanged [2021-10-22T01:55:56.397+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:56.397] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:57.398+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:57.398] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:58.398+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:58.398] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:55:59.399+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:55:59.399] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:00.400+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:00.400] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:01.401+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:01.401] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:02.401+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:02.401] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:03.402+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:03.402] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:04.403+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:04.403] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:05.411+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:05.411] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:06.266+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:56:06.266] clusterConfig unchanged [2021-10-22T01:56:06.404+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:06.404] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:07.404+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:07.404] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:08.404+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:08.404] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:09.405+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:09.405] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:10.406+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:10.406] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:11.407+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:11.407] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:12.407+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:12.407] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:13.408+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:13.408] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:14.408+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:14.408] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:15.409+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:15.409] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:16.416+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:16.416] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:16.516+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:56:16.516] clusterConfig unchanged [2021-10-22T01:56:17.411+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:17.411] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:18.411+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:18.411] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:18.563+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:56:18.563] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:56:18.565+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:18.565] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:56:18.565+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [01:56:18.565] Could not get version : [01:56:18.565] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:56:18.565+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [01:56:18.565] Error getting version for log lines. Skipping : [01:56:18.565] Could not get version : [01:56:18.565] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T01:56:19.412+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:19.412] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:20.413+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:20.413] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:21.414+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:21.414] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:22.414+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:22.414] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:23.414+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:23.414] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:24.415+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:24.415] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:25.422+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:25.422] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:26.416+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:26.416] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:26.658+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:56:26.658] clusterConfig unchanged [2021-10-22T01:56:27.417+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:27.417] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:28.419+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:28.419] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:29.418+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:29.418] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:30.418+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:30.418] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:31.418+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:31.418] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:32.419+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:32.419] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:33.419+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:33.419] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:34.419+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:34.419] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:35.420+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:35.420] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:36.427+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:36.427] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:36.799+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:56:36.799] clusterConfig unchanged [2021-10-22T01:56:37.421+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:37.421] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:38.421+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:38.421] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:39.421+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:39.421] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:40.422+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:40.422] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:41.369+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:56:41.369] memory:rss=55222272 alloc=10458408 totalalloc=8199935216 mallocs=104518897 frees=104436260 heapinuse=16171008 heapobjects=82637 gcpausetotalns=1564782722 gcpauselastns=61336 gcnum=1242 utimens=166551782000 stimens=62695139000 minflt=432900 majflt=405 [2021-10-22T01:56:41.422+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:41.422] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:42.422+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:42.422] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:43.424+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:43.424] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:44.348+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [01:56:44.348] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34751 (local=false) because it is not up [2021-10-22T01:56:44.429+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:44.429] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:45.052+0000] [.info] [src/director/director.go:computePlan:280] [01:56:45.052] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:56:45.052+0000] [.info] [src/director/director.go:tracef:794] [01:56:45.052] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:56:45.052+0000] [.info] [src/director/director.go:tracef:794] [01:56:45.052] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:56:45.052+0000] [.info] [src/director/director.go:tracef:794] [01:56:45.052] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:56:45.053+0000] [.info] [src/director/director.go:tracef:794] [01:56:45.052] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:56:45.053+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:56:45.053] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:56:45.202+0000] [.info] [src/director/director.go:tracef:794] [01:56:45.202] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:56:45.202+0000] [.info] [src/director/director.go:tracef:794] [01:56:45.202] because [2021-10-22T01:56:45.202+0000] [.info] [src/director/director.go:tracef:794] [01:56:45.202] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:56:45.202+0000] [.info] [src/director/director.go:tracef:794] [01:56:45.202] because ['currentState.CanBounceStop' = true] [2021-10-22T01:56:45.358+0000] [.info] [src/director/director.go:tracef:794] [01:56:45.358] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:56:45.358+0000] [.info] [src/director/director.go:tracef:794] [01:56:45.358] because [2021-10-22T01:56:45.358+0000] [.info] [src/director/director.go:tracef:794] [01:56:45.358] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:56:45.358+0000] [.info] [src/director/director.go:tracef:794] [01:56:45.358] because ['currentState.Up' = false] [2021-10-22T01:56:45.358+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:56:45.358] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:56:45.358+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:56:45.358] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:56:45.423+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:45.423] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:45.445+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:56:45.445] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:56:45.445+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:56:45.445] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:56:45.453+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:56:45.453] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:56:45.454+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:56:45.454] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:56:45.455+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:56:45.455] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:56:45.455+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:56:45.455] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:56:45.455+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:56:45.455] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:56:45.457+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:56:45.457] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:56:45.674+0000] [.info] [src/director/director.go:tracef:794] [01:56:45.674] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:56:45.674+0000] [.info] [src/director/director.go:tracef:794] [01:56:45.674] because ['currentState.Up' = false] [2021-10-22T01:56:45.877+0000] [.info] [src/director/director.go:tracef:794] [01:56:45.877] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:56:45.877+0000] [.info] [src/director/director.go:tracef:794] [01:56:45.877] because ['currentState.Up' = false] [2021-10-22T01:56:45.877+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:56:45.877] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:56:46.104+0000] [.info] [src/director/director.go:tracef:794] [01:56:46.104] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:56:46.105+0000] [.info] [src/director/director.go:tracef:794] [01:56:46.104] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:56:46.332+0000] [.info] [src/director/director.go:tracef:794] [01:56:46.332] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:56:46.332+0000] [.info] [src/director/director.go:tracef:794] [01:56:46.332] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:56:46.332+0000] [.info] [src/director/director.go:tracef:794] [01:56:46.332] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:56:46.332+0000] [.info] [src/director/director.go:tracef:794] [01:56:46.332] because ['currentState.Up' = false] [2021-10-22T01:56:46.332+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:56:46.332] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:56:46.332+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:56:46.332] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:56:46.365+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:56:46.365] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:56:46.366+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:56:46.366] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 44885 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:56:46.427+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:46.427] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:47.022+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:56:47.022] clusterConfig unchanged [2021-10-22T01:56:47.096+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:56:47.096] ...Finished start [pid=27820] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44885,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:56:47.230+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:56:47.230] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 133.608616ms. [2021-10-22T01:56:47.230+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:56:47.230] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44885 (local=false) [2021-10-22T01:56:47.230+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:56:47.230] Dropping existing documents from system.replset [2021-10-22T01:56:47.368+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:56:47.368] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:56:47.372+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:56:47.372] Inserting oplogTruncateAfterPoint document [2021-10-22T01:56:47.372+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:56:47.372] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:56:47.373+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:56:47.373] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:56:47.373+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:56:47.373] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:56:47.387+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:56:47.387] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:56:47.387+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:56:47.387] Setting the minvalid document to [{_id ObjectID("61721a5f523370002becc6f1")} {t -1} {ts {0 1}}] [2021-10-22T01:56:47.401+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:56:47.401] Inserted [{_id ObjectID("61721a5f523370002becc6f1")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:56:47.401+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:56:47.401] Inserting new replica set configuration document. [2021-10-22T01:56:47.401+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:56:47.401] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:56:47.415+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:56:47.415] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:56:47.426+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:47.426] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:47.427+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:56:47.427] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:56:47.427+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:56:47.427] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:56:47.427+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:56:47.427] Initiating stopEphemeralMongo [2021-10-22T01:56:47.427+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:56:47.427] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44885 (local=false) [2021-10-22T01:56:47.427+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:56:47.427] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44885 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:56:47.856+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:56:47.856] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.108016ms. [2021-10-22T01:56:47.856+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:56:47.856] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44885 (local=false) [2021-10-22T01:56:47.856+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:56:47.856] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.601µs. [2021-10-22T01:56:47.857+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:56:47.857] Successfully finished running stopEphemeralMongo [2021-10-22T01:56:47.857+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:56:47.857] Successfully stopped existing ephemeral mongod. [2021-10-22T01:56:47.857+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:56:47.857] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:56:47.857+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:56:47.857] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:56:47.890+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:56:47.890] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:56:47.891+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:56:47.891] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37083 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:56:48.113+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:56:48.113] clusterConfig unchanged [2021-10-22T01:56:48.429+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:48.428] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:49.131+0000] [.error] [src/action/start.go:func1:93] [103] [01:56:49.131] Error starting mongod : [01:56:49.131] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37083,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37083 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 29351 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:56:47.943+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:56:47.943+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:56:47.943+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37083,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:56:47.945+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:56:47.946+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:56:47.947+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:56:47.964+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:56:47.964+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:56:47.975+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:56:48.019+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:56:48.019+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:56:48.595+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867808:595670][29351:0x7f990112ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:56:48.700+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867808:700379][29351:0x7f990112ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:56:48.815+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867808:815518][29351:0x7f990112ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T01:56:48.935+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867808:935373][29351:0x7f990112ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:56:49.012+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867809:12576][29351:0x7f990112ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:56:49.079+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867809:79839][29351:0x7f990112ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:56:49.079+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867809:79916][29351:0x7f990112ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:56:49.081+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"in...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:56:49.131+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:56:49.131] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37083 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:56:49.131] Error starting mongod : [01:56:49.131] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37083,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37083 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 29351 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:56:47.943+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:56:47.943+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:56:47.943+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37083,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:56:47.945+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:56:47.946+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:56:47.947+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:56:47.964+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:56:47.964+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:56:47.975+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:56:48.019+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:56:48.019+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:56:48.595+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867808:595670][29351:0x7f990112ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:56:48.700+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867808:700379][29351:0x7f990112ebc0], txn-recover:...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:56:49.131+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:56:49.131] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:56:49.131] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37083 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:56:49.131] Error starting mongod : [01:56:49.131] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37083,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37083 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 29351 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:56:47.943+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:56:47.943+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:56:47.943+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37083,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:56:47.945+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:56:47.946+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:56:47.947+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:56:47.964+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:56:47.964+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:56:47.975+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:56:48.019+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:56:48.019+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:56:48.595+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867808:595670][29351:0x7f990112ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:56:48.700+00:00"},"s":"I", "c":"STORAGE", "id":22430, ... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:56:49.132+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:56:49.132] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:56:49.132+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:56:49.132] Initiating stopEphemeralMongo [2021-10-22T01:56:49.132+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:56:49.132] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:56:49.132+0000] [.error] [src/director/director.go:executePlan:962] [01:56:49.132] Failed to apply action. Result = : [01:56:49.131] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:56:49.131] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37083 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:56:49.131] Error starting mongod : [01:56:49.131] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37083,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37083 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 29351 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:56:47.943+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:56:47.943+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:56:47.943+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37083,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:56:47.945+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:56:47.946+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:56:47.947+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:56:47.964+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:56:47.964+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:56:47.975+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:56:48.019+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:56:48.019+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:56:48.595+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867808:595670][29351:0x7f990112ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} ... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:56:49.132+0000] [.error] [src/director/director.go:planAndExecute:567] [01:56:49.132] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:56:49.132] Failed to apply action. Result = : [01:56:49.131] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:56:49.131] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37083 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:56:49.131] Error starting mongod : [01:56:49.131] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37083,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37083 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 29351 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:56:47.943+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:56:47.943+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:56:47.943+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37083,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:56:47.945+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:56:47.946+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:56:47.947+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:56:47.964+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:56:47.964+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:56:47.975+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:56:48.019+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:56:48.019+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:56:48.595+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger mess... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:56:49.132+0000] [.error] [src/director/director.go:mainLoop:399] [01:56:49.132] Failed to planAndExecute : [01:56:49.132] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:56:49.132] Failed to apply action. Result = : [01:56:49.131] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:56:49.131] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37083 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:56:49.131] Error starting mongod : [01:56:49.131] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37083,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37083 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 29351 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:56:47.943+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:56:47.943+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:56:47.943+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37083,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:56:47.945+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:56:47.946+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:56:47.947+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:56:47.964+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:56:47.964+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:56:47.975+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:56:48.019+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:56:48.019+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:56:48.595+00:00"},"s":"I", "... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:56:49.375+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:56:49.375] clusterConfig unchanged [2021-10-22T01:56:49.428+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:49.428] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:50.431+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:50.431] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:51.429+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:51.429] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:52.430+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:52.430] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:53.430+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:53.430] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:54.432+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:54.432] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:55.432+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:55.432] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:56.430+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:56.430] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:57.434+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:57.434] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:58.143+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:56:58.143] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.68604311s [2021-10-22T01:56:58.431+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:58.431] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:59.431+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:56:59.431] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:56:59.485+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:56:59.485] clusterConfig unchanged [2021-10-22T01:57:00.439+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:00.439] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:01.433+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:01.433] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:02.434+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:02.434] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:03.435+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:03.435] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:04.435+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:04.435] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:05.435+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:05.435] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:06.436+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:06.435] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:07.438+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:07.437] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:08.437+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:08.437] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:09.438+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:09.438] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:09.597+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:57:09.597] clusterConfig unchanged [2021-10-22T01:57:10.438+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:10.438] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:11.446+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:11.446] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:12.439+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:12.439] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:13.440+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:13.440] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:14.440+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:14.440] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:15.441+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:15.441] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:16.441+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:16.441] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:17.442+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:17.442] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:18.443+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:18.443] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:19.443+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:19.443] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:19.719+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:57:19.719] clusterConfig unchanged [2021-10-22T01:57:20.444+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:20.444] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:21.449+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:21.449] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:22.453+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:22.453] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:23.447+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:23.447] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:24.447+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:24.447] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:25.448+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:25.448] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:26.448+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:26.448] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:27.448+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:27.448] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:27.567+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:57:27.567] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:57:27.568+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [01:57:27.568] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T01:57:28.450+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:28.450] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:29.449+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:29.449] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:29.891+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:57:29.891] clusterConfig unchanged [2021-10-22T01:57:30.450+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:30.450] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:31.451+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:31.451] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:32.452+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:32.452] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:33.452+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:33.452] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:34.453+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:34.453] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:35.453+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:35.453] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:36.454+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:36.454] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:37.456+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [01:57:37.456] lsblk cache invalidated, running lsblk command [2021-10-22T01:57:37.457+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:37.457] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:38.455+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:38.455] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:39.455+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:39.455] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:39.993+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:57:39.993] clusterConfig unchanged [2021-10-22T01:57:40.456+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:40.456] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:41.370+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:57:41.370] memory:rss=54751232 alloc=11971360 totalalloc=8291044128 mallocs=105679357 frees=105580450 heapinuse=16695296 heapobjects=98907 gcpausetotalns=1565760235 gcpauselastns=50683 gcnum=1254 utimens=168913457000 stimens=63694945000 minflt=438046 majflt=405 [2021-10-22T01:57:41.456+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:41.456] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:42.457+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:42.457] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:43.458+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:43.458] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:44.459+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:44.459] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:45.460+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:45.460] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:46.460+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:46.460] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:47.461+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:47.461] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:48.462+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:48.462] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:49.311+0000] [.info] [src/director/director.go:computePlan:280] [01:57:49.311] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:57:49.311+0000] [.info] [src/director/director.go:tracef:794] [01:57:49.311] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:57:49.311+0000] [.info] [src/director/director.go:tracef:794] [01:57:49.311] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:57:49.311+0000] [.info] [src/director/director.go:tracef:794] [01:57:49.311] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:57:49.311+0000] [.info] [src/director/director.go:tracef:794] [01:57:49.311] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:57:49.312+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:57:49.312] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:57:49.466+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:49.466] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:49.474+0000] [.info] [src/director/director.go:tracef:794] [01:57:49.474] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:57:49.474+0000] [.info] [src/director/director.go:tracef:794] [01:57:49.474] because [2021-10-22T01:57:49.474+0000] [.info] [src/director/director.go:tracef:794] [01:57:49.474] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:57:49.474+0000] [.info] [src/director/director.go:tracef:794] [01:57:49.474] because ['currentState.CanBounceStop' = true] [2021-10-22T01:57:49.626+0000] [.info] [src/director/director.go:tracef:794] [01:57:49.626] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:57:49.626+0000] [.info] [src/director/director.go:tracef:794] [01:57:49.626] because [2021-10-22T01:57:49.626+0000] [.info] [src/director/director.go:tracef:794] [01:57:49.626] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:57:49.626+0000] [.info] [src/director/director.go:tracef:794] [01:57:49.626] because ['currentState.Up' = false] [2021-10-22T01:57:49.626+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:57:49.626] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:57:49.626+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:57:49.626] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:57:49.710+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:57:49.710] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:57:49.718+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:57:49.718] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:57:49.719+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:57:49.719] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:57:49.720+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:57:49.720] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:57:49.720+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:57:49.720] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:57:49.720+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:57:49.720] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:57:49.721+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:57:49.721] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:57:49.944+0000] [.info] [src/director/director.go:tracef:794] [01:57:49.944] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:57:49.944+0000] [.info] [src/director/director.go:tracef:794] [01:57:49.944] because ['currentState.Up' = false] [2021-10-22T01:57:50.161+0000] [.info] [src/director/director.go:tracef:794] [01:57:50.161] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:57:50.161+0000] [.info] [src/director/director.go:tracef:794] [01:57:50.161] because ['currentState.Up' = false] [2021-10-22T01:57:50.162+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:57:50.162] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:57:50.253+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:57:50.253] clusterConfig unchanged [2021-10-22T01:57:50.390+0000] [.info] [src/director/director.go:tracef:794] [01:57:50.390] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:57:50.390+0000] [.info] [src/director/director.go:tracef:794] [01:57:50.390] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:57:50.467+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:50.467] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:50.617+0000] [.info] [src/director/director.go:tracef:794] [01:57:50.617] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:57:50.617+0000] [.info] [src/director/director.go:tracef:794] [01:57:50.617] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:57:50.617+0000] [.info] [src/director/director.go:tracef:794] [01:57:50.617] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:57:50.617+0000] [.info] [src/director/director.go:tracef:794] [01:57:50.617] because ['currentState.Up' = false] [2021-10-22T01:57:50.618+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:57:50.617] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:57:50.618+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:57:50.618] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:57:50.650+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:57:50.650] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:57:50.650+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:57:50.650] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51722 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:57:51.364+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:57:51.364] ...Finished start [pid=48535] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51722,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:57:51.394+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:57:51.394] clusterConfig unchanged [2021-10-22T01:57:51.468+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:51.468] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:51.500+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:57:51.499] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 134.774066ms. [2021-10-22T01:57:51.500+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:57:51.500] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51722 (local=false) [2021-10-22T01:57:51.500+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:57:51.500] Dropping existing documents from system.replset [2021-10-22T01:57:51.655+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:57:51.655] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:57:51.660+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:57:51.660] Inserting oplogTruncateAfterPoint document [2021-10-22T01:57:51.660+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:57:51.660] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:57:51.661+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:57:51.661] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:57:51.661+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:57:51.661] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:57:51.674+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:57:51.674] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:57:51.674+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:57:51.674] Setting the minvalid document to [{_id ObjectID("61721a9f523370002becc6ff")} {t -1} {ts {0 1}}] [2021-10-22T01:57:51.689+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:57:51.689] Inserted [{_id ObjectID("61721a9f523370002becc6ff")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:57:51.689+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:57:51.689] Inserting new replica set configuration document. [2021-10-22T01:57:51.689+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:57:51.689] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:57:51.701+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:57:51.701] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:57:51.714+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:57:51.714] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:57:51.714+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:57:51.714] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:57:51.714+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:57:51.714] Initiating stopEphemeralMongo [2021-10-22T01:57:51.714+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:57:51.714] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51722 (local=false) [2021-10-22T01:57:51.714+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:57:51.714] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51722 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:57:52.079+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:57:52.079] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.192134ms. [2021-10-22T01:57:52.079+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:57:52.079] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51722 (local=false) [2021-10-22T01:57:52.079+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:57:52.079] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.671µs. [2021-10-22T01:57:52.079+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:57:52.079] Successfully finished running stopEphemeralMongo [2021-10-22T01:57:52.079+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:57:52.079] Successfully stopped existing ephemeral mongod. [2021-10-22T01:57:52.079+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:57:52.079] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:57:52.079+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:57:52.079] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:57:52.113+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:57:52.113] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:57:52.114+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:57:52.114] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41176 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:57:52.472+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:52.472] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:52.624+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:57:52.624] clusterConfig unchanged [2021-10-22T01:57:53.351+0000] [.error] [src/action/start.go:func1:93] [103] [01:57:53.351] Error starting mongod : [01:57:53.351] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41176,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41176 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:57:52.146Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 49919 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:57:52.166+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:57:52.166+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:57:52.166+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41176,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:57:52.168+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:57:52.171+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:57:52.172+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:57:52.193+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:57:52.193+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:57:52.193+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:57:52.240+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:57:52.240+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:57:52.831+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867872:831211][49919:0x7f7a6521cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:57:52.940+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867872:940518][49919:0x7f7a6521cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:57:53.042+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867873:42731][49919:0x7f7a6521cbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35456 to 2/256"}} {"t":{"$date":"2021-10-22T01:57:53.154+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867873:154913][49919:0x7f7a6521cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:57:53.235+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867873:235673][49919:0x7f7a6521cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:57:53.300+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867873:300552][49919:0x7f7a6521cbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:57:53.300+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867873:300629][49919:0x7f7a6521cbc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T01:57:53.351+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:57:53.351] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:41176 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:57:53.351] Error starting mongod : [01:57:53.351] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41176,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41176 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:57:52.146Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 49919 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:57:52.166+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:57:52.166+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:57:52.166+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41176,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:57:52.168+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:57:52.171+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:57:52.172+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:57:52.193+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:57:52.193+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:57:52.193+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:57:52.240+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:57:52.240+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:57:52.831+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867872:831211][49919:0x7f7a6521cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:57:52.940+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:57:53.351+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:57:53.351] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:57:53.351] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:41176 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:57:53.351] Error starting mongod : [01:57:53.351] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41176,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41176 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:57:52.146Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 49919 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:57:52.166+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:57:52.166+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:57:52.166+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41176,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:57:52.168+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:57:52.171+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:57:52.172+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:57:52.193+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:57:52.193+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:57:52.193+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:57:52.240+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:57:52.240+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:57:52.831+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867872:831211][49919:0x7f7a6521cbc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:57:53.351+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:57:53.351] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:57:53.351+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:57:53.351] Initiating stopEphemeralMongo [2021-10-22T01:57:53.352+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:57:53.351] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:57:53.352+0000] [.error] [src/director/director.go:executePlan:962] [01:57:53.352] Failed to apply action. Result = : [01:57:53.351] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:57:53.351] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:41176 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:57:53.351] Error starting mongod : [01:57:53.351] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41176,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41176 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:57:52.146Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 49919 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:57:52.166+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:57:52.166+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:57:52.166+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41176,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:57:52.168+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:57:52.171+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:57:52.172+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:57:52.193+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:57:52.193+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:57:52.193+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:57:52.240+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:57:52.240+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:57:52.831+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:57:53.352+0000] [.error] [src/director/director.go:planAndExecute:567] [01:57:53.352] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:57:53.352] Failed to apply action. Result = : [01:57:53.351] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:57:53.351] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:41176 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:57:53.351] Error starting mongod : [01:57:53.351] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41176,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41176 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:57:52.146Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 49919 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:57:52.166+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:57:52.166+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:57:52.166+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41176,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:57:52.168+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:57:52.171+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:57:52.172+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:57:52.193+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:57:52.193+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:57:52.193+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:57:52.240+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:57:52.240+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:57:53.352+0000] [.error] [src/director/director.go:mainLoop:399] [01:57:53.352] Failed to planAndExecute : [01:57:53.352] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:57:53.352] Failed to apply action. Result = : [01:57:53.351] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:57:53.351] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:41176 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:57:53.351] Error starting mongod : [01:57:53.351] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41176,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41176 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T01:57:52.146Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 49919 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:57:52.166+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:57:52.166+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:57:52.166+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41176,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:57:52.168+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:57:52.171+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:57:52.172+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:57:52.193+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:57:52.193+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:57:52.193+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:57:52.240+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:57:52.240+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:57:53.469+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:53.469] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:53.856+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:57:53.856] clusterConfig unchanged [2021-10-22T01:57:54.469+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:54.469] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:55.469+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:55.469] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:56.468+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:56.468] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:57.470+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:57.470] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:58.472+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:58.472] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:57:59.472+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:57:59.472] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:00.473+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:00.473] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:01.474+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:01.474] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:02.440+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:58:02.440] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.718500428s [2021-10-22T01:58:02.471+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:02.471] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:03.471+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:03.471] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:03.967+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:58:03.967] clusterConfig unchanged [2021-10-22T01:58:04.471+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:04.471] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:05.472+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:05.472] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:06.473+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:06.473] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:07.480+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:07.480] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:08.475+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:08.475] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:09.476+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:09.476] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:10.476+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:10.476] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:11.480+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:11.480] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:12.479+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:12.479] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:13.478+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:13.478] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:14.138+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:58:14.138] clusterConfig unchanged [2021-10-22T01:58:14.478+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:14.478] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:15.478+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:15.478] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:16.479+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:16.479] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:17.480+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:17.480] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:18.481+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:18.481] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:19.488+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:19.488] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:20.481+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:20.481] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:21.481+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:21.481] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:22.482+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:22.482] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:23.483+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:23.483] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:24.280+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:58:24.280] clusterConfig unchanged [2021-10-22T01:58:24.483+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:24.483] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:25.484+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:25.484] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:26.505+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:26.505] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:27.485+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:27.485] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:28.485+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:28.485] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:29.486+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:29.486] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:30.487+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:30.487] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:31.487+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:31.487] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:32.488+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:32.488] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:33.489+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:33.489] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:34.421+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:58:34.421] clusterConfig unchanged [2021-10-22T01:58:34.489+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:34.489] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:35.489+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:35.489] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:36.489+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:36.489] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:37.490+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:37.490] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:38.491+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:38.491] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:39.491+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:39.491] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:40.492+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:40.492] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:41.371+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:58:41.371] memory:rss=55377920 alloc=7842560 totalalloc=8381793240 mallocs=106841461 frees=106789300 heapinuse=15687680 heapobjects=52161 gcpausetotalns=1567031529 gcpauselastns=59366 gcnum=1267 utimens=171100560000 stimens=64616247000 minflt=442546 majflt=405 [2021-10-22T01:58:41.493+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:41.493] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:42.496+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:42.496] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:43.493+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:43.493] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:44.496+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:44.496] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:44.524+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:58:44.524] clusterConfig unchanged [2021-10-22T01:58:45.494+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:45.494] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:46.495+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:46.495] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:47.496+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:47.496] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:48.496+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:48.496] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:49.504+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:49.504] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:50.497+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:50.497] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:51.498+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:51.498] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:52.499+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:52.499] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:53.501+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:53.501] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:53.518+0000] [.info] [src/director/director.go:computePlan:280] [01:58:53.518] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:58:53.518+0000] [.info] [src/director/director.go:tracef:794] [01:58:53.518] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:58:53.518+0000] [.info] [src/director/director.go:tracef:794] [01:58:53.518] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:58:53.518+0000] [.info] [src/director/director.go:tracef:794] [01:58:53.518] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:58:53.518+0000] [.info] [src/director/director.go:tracef:794] [01:58:53.518] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:58:53.518+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:58:53.518] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:58:53.679+0000] [.info] [src/director/director.go:tracef:794] [01:58:53.679] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:58:53.679+0000] [.info] [src/director/director.go:tracef:794] [01:58:53.679] because [2021-10-22T01:58:53.679+0000] [.info] [src/director/director.go:tracef:794] [01:58:53.679] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:58:53.679+0000] [.info] [src/director/director.go:tracef:794] [01:58:53.679] because ['currentState.CanBounceStop' = true] [2021-10-22T01:58:53.829+0000] [.info] [src/director/director.go:tracef:794] [01:58:53.829] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:58:53.829+0000] [.info] [src/director/director.go:tracef:794] [01:58:53.829] because [2021-10-22T01:58:53.829+0000] [.info] [src/director/director.go:tracef:794] [01:58:53.829] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:58:53.829+0000] [.info] [src/director/director.go:tracef:794] [01:58:53.829] because ['currentState.Up' = false] [2021-10-22T01:58:53.829+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:58:53.829] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:58:53.829+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:58:53.829] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:58:53.919+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:58:53.919] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:58:53.927+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:58:53.927] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:58:53.927+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:58:53.927] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:58:53.928+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:58:53.928] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:58:53.929+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:58:53.929] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:58:53.929+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:58:53.929] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:58:53.929+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:58:53.929] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:58:53.930+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:58:53.930] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:58:54.164+0000] [.info] [src/director/director.go:tracef:794] [01:58:54.164] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:58:54.164+0000] [.info] [src/director/director.go:tracef:794] [01:58:54.164] because ['currentState.Up' = false] [2021-10-22T01:58:54.439+0000] [.info] [src/director/director.go:tracef:794] [01:58:54.439] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:58:54.439+0000] [.info] [src/director/director.go:tracef:794] [01:58:54.439] because ['currentState.Up' = false] [2021-10-22T01:58:54.439+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:58:54.439] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:58:54.538+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:54.538] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:54.664+0000] [.info] [src/director/director.go:tracef:794] [01:58:54.664] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:58:54.664+0000] [.info] [src/director/director.go:tracef:794] [01:58:54.664] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:58:54.706+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:58:54.706] clusterConfig unchanged [2021-10-22T01:58:54.873+0000] [.info] [src/director/director.go:tracef:794] [01:58:54.873] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:58:54.873+0000] [.info] [src/director/director.go:tracef:794] [01:58:54.873] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:58:54.873+0000] [.info] [src/director/director.go:tracef:794] [01:58:54.873] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:58:54.873+0000] [.info] [src/director/director.go:tracef:794] [01:58:54.873] because ['currentState.Up' = false] [2021-10-22T01:58:54.873+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:58:54.873] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:58:54.873+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:58:54.873] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:58:54.904+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:58:54.904] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:58:54.904+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:58:54.904] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38858 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:58:55.505+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:55.505] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:55.644+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:58:55.644] ...Finished start [pid=69436] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38858,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:58:55.768+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:58:55.768] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 123.16627ms. [2021-10-22T01:58:55.768+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:58:55.768] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38858 (local=false) [2021-10-22T01:58:55.768+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:58:55.768] Dropping existing documents from system.replset [2021-10-22T01:58:55.818+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:58:55.818] clusterConfig unchanged [2021-10-22T01:58:55.927+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [01:58:55.927] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T01:58:55.931+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [01:58:55.931] Inserting oplogTruncateAfterPoint document [2021-10-22T01:58:55.931+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [01:58:55.931] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T01:58:55.932+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [01:58:55.932] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T01:58:55.932+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [01:58:55.932] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T01:58:55.945+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:58:55.945] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T01:58:55.945+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [01:58:55.945] Setting the minvalid document to [{_id ObjectID("61721adf523370002becc70d")} {t -1} {ts {0 1}}] [2021-10-22T01:58:55.958+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:58:55.958] Inserted [{_id ObjectID("61721adf523370002becc70d")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T01:58:55.958+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [01:58:55.958] Inserting new replica set configuration document. [2021-10-22T01:58:55.958+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [01:58:55.958] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T01:58:55.970+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [01:58:55.970] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T01:58:55.984+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [01:58:55.984] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T01:58:55.984+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [01:58:55.984] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T01:58:55.984+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:58:55.984] Initiating stopEphemeralMongo [2021-10-22T01:58:55.984+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [01:58:55.984] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38858 (local=false) [2021-10-22T01:58:55.984+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [01:58:55.984] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38858 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T01:58:56.373+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:58:56.373] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.114078ms. [2021-10-22T01:58:56.373+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [01:58:56.373] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38858 (local=false) [2021-10-22T01:58:56.373+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:58:56.373] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 50.825µs. [2021-10-22T01:58:56.374+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [01:58:56.374] Successfully finished running stopEphemeralMongo [2021-10-22T01:58:56.374+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [01:58:56.374] Successfully stopped existing ephemeral mongod. [2021-10-22T01:58:56.374+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [01:58:56.374] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T01:58:56.374+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [01:58:56.374] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:58:56.406+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:58:56.406] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:58:56.407+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:58:56.407] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60091 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:58:56.504+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:56.504] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:56.921+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:58:56.921] clusterConfig unchanged [2021-10-22T01:58:57.503+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:57.503] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:57.638+0000] [.error] [src/action/start.go:func1:93] [103] [01:58:57.638] Error starting mongod : [01:58:57.638] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60091,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60091 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 71010 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:58:56.482+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:58:56.482+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:58:56.482+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60091,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:58:56.484+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:58:56.486+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:58:56.486+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:58:56.505+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:58:56.505+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:58:56.506+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:58:56.551+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:58:56.551+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:58:57.112+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867937:112800][71010:0x7ff51df52bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:58:57.218+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867937:218882][71010:0x7ff51df52bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:58:57.322+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867937:322853][71010:0x7ff51df52bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35456 to 2/256"}} {"t":{"$date":"2021-10-22T01:58:57.439+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867937:439218][71010:0x7ff51df52bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:58:57.507+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867937:507935][71010:0x7ff51df52bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T01:58:57.587+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867937:587409][71010:0x7ff51df52bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:58:57.587+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867937:587483][71010:0x7ff51df52bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T01:58:57.589+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:58:57.638+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [01:58:57.638] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60091 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:58:57.638] Error starting mongod : [01:58:57.638] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60091,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60091 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 71010 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:58:56.482+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:58:56.482+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:58:56.482+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60091,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:58:56.484+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:58:56.486+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:58:56.486+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:58:56.505+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:58:56.505+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:58:56.506+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:58:56.551+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:58:56.551+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:58:57.112+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867937:112800][71010:0x7ff51df52bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:58:57.218+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867937:218882][71010:0x7ff51df52bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T01:58:57.638+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [01:58:57.638] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:58:57.638] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60091 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:58:57.638] Error starting mongod : [01:58:57.638] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60091,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60091 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 71010 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:58:56.482+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:58:56.482+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:58:56.482+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60091,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:58:56.484+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:58:56.486+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:58:56.486+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:58:56.505+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:58:56.505+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:58:56.506+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:58:56.551+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:58:56.551+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:58:57.112+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867937:112800][71010:0x7ff51df52bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T01:58:57.218+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:58:57.638+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:58:57.638] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:58:57.638+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [01:58:57.638] Initiating stopEphemeralMongo [2021-10-22T01:58:57.639+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [01:58:57.638] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T01:58:57.639+0000] [.error] [src/director/director.go:executePlan:962] [01:58:57.639] Failed to apply action. Result = : [01:58:57.638] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:58:57.638] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60091 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:58:57.638] Error starting mongod : [01:58:57.638] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60091,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60091 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 71010 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:58:56.482+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:58:56.482+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:58:56.482+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60091,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:58:56.484+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:58:56.486+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:58:56.486+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:58:56.505+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:58:56.505+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:58:56.506+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:58:56.551+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:58:56.551+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:58:57.112+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634867937:112800][71010:0x7ff51df52bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:58:57.639+0000] [.error] [src/director/director.go:planAndExecute:567] [01:58:57.639] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:58:57.639] Failed to apply action. Result = : [01:58:57.638] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:58:57.638] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60091 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:58:57.638] Error starting mongod : [01:58:57.638] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60091,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60091 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 71010 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:58:56.482+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:58:56.482+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:58:56.482+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60091,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:58:56.484+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:58:56.486+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:58:56.486+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:58:56.505+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:58:56.505+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:58:56.506+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:58:56.551+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:58:56.551+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:58:57.112+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:58:57.640+0000] [.error] [src/director/director.go:mainLoop:399] [01:58:57.640] Failed to planAndExecute : [01:58:57.639] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [01:58:57.639] Failed to apply action. Result = : [01:58:57.638] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [01:58:57.638] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60091 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [01:58:57.638] Error starting mongod : [01:58:57.638] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60091,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60091 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 71010 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T01:58:56.482+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T01:58:56.482+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T01:58:56.482+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60091,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T01:58:56.484+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:58:56.486+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T01:58:56.486+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T01:58:56.505+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:58:56.505+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:58:56.506+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T01:58:56.551+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T01:58:56.551+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T01:58:57.112+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T01:58:58.172+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:58:58.172] clusterConfig unchanged [2021-10-22T01:58:58.504+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:58.504] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:58:59.502+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:58:59.502] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:00.504+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:00.504] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:01.505+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:01.505] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:02.507+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:02.507] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:03.506+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:03.506] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:04.505+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:04.505] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:05.509+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:05.509] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:06.507+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:06.506] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:06.615+0000] [.info] [src/util/io.go:PreWarmDisk:56] [01:59:06.614] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.68400579s [2021-10-22T01:59:07.507+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:07.507] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:08.273+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:59:08.273] clusterConfig unchanged [2021-10-22T01:59:08.508+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:08.508] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:09.508+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:09.508] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:10.509+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:10.509] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:11.510+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:11.510] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:12.510+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:12.510] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:13.511+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:13.511] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:14.516+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:14.516] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:15.511+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:15.511] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:16.511+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:16.511] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:17.519+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:17.519] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:18.366+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:59:18.366] clusterConfig unchanged [2021-10-22T01:59:18.512+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:18.512] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:19.513+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:19.513] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:20.514+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:20.514] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:21.514+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:21.514] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:22.514+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:22.514] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:23.515+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:23.515] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:24.515+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:24.515] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:25.516+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:25.516] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:26.517+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:26.517] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:27.518+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:27.518] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:28.467+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:59:28.467] clusterConfig unchanged [2021-10-22T01:59:28.518+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:28.518] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:29.526+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:29.526] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:30.519+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:30.519] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:31.519+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:31.519] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:32.520+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:32.520] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:33.521+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:33.521] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:34.522+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:34.522] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:35.522+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:35.522] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:36.524+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:36.523] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:37.524+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:37.524] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:38.525+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:38.525] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:38.610+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:59:38.610] clusterConfig unchanged [2021-10-22T01:59:39.525+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:39.525] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:40.526+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:40.526] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:41.372+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [01:59:41.372] memory:rss=55623680 alloc=14902368 totalalloc=8471256912 mallocs=107997938 frees=107864798 heapinuse=17940480 heapobjects=133140 gcpausetotalns=1568576659 gcpauselastns=90842 gcnum=1278 utimens=173311019000 stimens=65566771000 minflt=447473 majflt=405 [2021-10-22T01:59:41.527+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:41.527] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:42.527+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:42.527] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:43.527+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:43.527] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:44.356+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [01:59:44.356] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:60091 (local=false) because it is not up [2021-10-22T01:59:44.528+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:44.528] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:45.528+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:45.528] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:46.530+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:46.530] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:47.530+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:47.530] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:48.534+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:48.534] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:48.710+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:59:48.710] clusterConfig unchanged [2021-10-22T01:59:49.531+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:49.531] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:50.532+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:50.532] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:51.532+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:51.532] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:52.532+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:52.532] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:53.533+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:53.533] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:54.535+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:54.535] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:55.535+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:55.535] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:56.536+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:56.536] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:57.537+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:57.537] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:57.804+0000] [.info] [src/director/director.go:computePlan:280] [01:59:57.804] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T01:59:57.804+0000] [.info] [src/director/director.go:tracef:794] [01:59:57.804] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:59:57.804+0000] [.info] [src/director/director.go:tracef:794] [01:59:57.804] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T01:59:57.805+0000] [.info] [src/director/director.go:tracef:794] [01:59:57.805] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:59:57.805+0000] [.info] [src/director/director.go:tracef:794] [01:59:57.805] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T01:59:57.805+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:59:57.805] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:59:57.957+0000] [.info] [src/director/director.go:tracef:794] [01:59:57.957] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:59:57.957+0000] [.info] [src/director/director.go:tracef:794] [01:59:57.957] because [2021-10-22T01:59:57.957+0000] [.info] [src/director/director.go:tracef:794] [01:59:57.957] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:59:57.957+0000] [.info] [src/director/director.go:tracef:794] [01:59:57.957] because ['currentState.CanBounceStop' = true] [2021-10-22T01:59:58.124+0000] [.info] [src/director/director.go:tracef:794] [01:59:58.124] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:59:58.124+0000] [.info] [src/director/director.go:tracef:794] [01:59:58.124] because [2021-10-22T01:59:58.124+0000] [.info] [src/director/director.go:tracef:794] [01:59:58.124] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:59:58.124+0000] [.info] [src/director/director.go:tracef:794] [01:59:58.124] because ['currentState.Up' = false] [2021-10-22T01:59:58.124+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [01:59:58.124] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:59:58.124+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [01:59:58.124] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T01:59:58.203+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:59:58.203] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T01:59:58.211+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:59:58.211] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T01:59:58.211+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [01:59:58.211] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T01:59:58.211+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:59:58.211] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T01:59:58.213+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [01:59:58.213] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:59:58.213+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [01:59:58.213] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T01:59:58.213+0000] [.info] [src/util/io.go:PreWarmDisk:29] [01:59:58.213] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:59:58.214+0000] [.info] [src/util/io.go:PreWarmDisk:45] [01:59:58.214] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T01:59:58.417+0000] [.info] [src/director/director.go:tracef:794] [01:59:58.417] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:59:58.417+0000] [.info] [src/director/director.go:tracef:794] [01:59:58.417] because ['currentState.Up' = false] [2021-10-22T01:59:58.539+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:58.539] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:58.619+0000] [.info] [src/director/director.go:tracef:794] [01:59:58.619] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:59:58.619+0000] [.info] [src/director/director.go:tracef:794] [01:59:58.619] because ['currentState.Up' = false] [2021-10-22T01:59:58.619+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [01:59:58.619] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T01:59:58.839+0000] [.info] [src/director/director.go:tracef:794] [01:59:58.839] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T01:59:58.839+0000] [.info] [src/director/director.go:tracef:794] [01:59:58.839] because ['desiredState.ProcessType' = mongod] [2021-10-22T01:59:58.916+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [01:59:58.916] clusterConfig unchanged [2021-10-22T01:59:59.047+0000] [.info] [src/director/director.go:tracef:794] [01:59:59.047] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:59:59.047+0000] [.info] [src/director/director.go:tracef:794] [01:59:59.047] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T01:59:59.047+0000] [.info] [src/director/director.go:tracef:794] [01:59:59.047] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T01:59:59.047+0000] [.info] [src/director/director.go:tracef:794] [01:59:59.047] because ['currentState.Up' = false] [2021-10-22T01:59:59.048+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [01:59:59.048] Initiating startEphemeralMongoAsStandalone [2021-10-22T01:59:59.048+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [01:59:59.048] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T01:59:59.083+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [01:59:59.083] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T01:59:59.083+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [01:59:59.083] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58539 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T01:59:59.548+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [01:59:59.548] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T01:59:59.798+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [01:59:59.798] ...Finished start [pid=16003] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58539,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T01:59:59.961+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [01:59:59.961] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 162.638695ms. [2021-10-22T01:59:59.961+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [01:59:59.961] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:58539 (local=false) [2021-10-22T01:59:59.961+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [01:59:59.961] Dropping existing documents from system.replset [2021-10-22T02:00:00.063+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:00:00.063] clusterConfig unchanged [2021-10-22T02:00:00.143+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:00:00.143] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:00:00.148+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:00:00.148] Inserting oplogTruncateAfterPoint document [2021-10-22T02:00:00.148+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:00:00.148] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:00:00.150+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:00:00.149] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:00:00.150+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:00:00.150] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:00:00.164+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:00:00.164] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:00:00.164+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:00:00.164] Setting the minvalid document to [{_id ObjectID("61721b20523370002becc71b")} {t -1} {ts {0 1}}] [2021-10-22T02:00:00.179+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:00:00.179] Inserted [{_id ObjectID("61721b20523370002becc71b")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:00:00.180+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:00:00.180] Inserting new replica set configuration document. [2021-10-22T02:00:00.180+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:00:00.180] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:00:00.193+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:00:00.193] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:00:00.246+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:00:00.246] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:00:00.246+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:00:00.246] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:00:00.246+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:00:00.246] Initiating stopEphemeralMongo [2021-10-22T02:00:00.246+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:00:00.246] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:58539 (local=false) [2021-10-22T02:00:00.246+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:00:00.246] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:58539 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:00:00.546+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:00.546] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:00.678+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:00:00.678] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.079663ms. [2021-10-22T02:00:00.678+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:00:00.678] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:58539 (local=false) [2021-10-22T02:00:00.678+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:00:00.678] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.002µs. [2021-10-22T02:00:00.678+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:00:00.678] Successfully finished running stopEphemeralMongo [2021-10-22T02:00:00.678+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:00:00.678] Successfully stopped existing ephemeral mongod. [2021-10-22T02:00:00.678+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:00:00.678] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:00:00.678+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:00:00.678] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:00:00.711+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:00:00.711] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:00:00.712+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:00:00.712] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55891 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:00:01.196+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:00:01.196] clusterConfig unchanged [2021-10-22T02:00:01.548+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:01.548] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:01.977+0000] [.error] [src/action/start.go:func1:93] [103] [02:00:01.977] Error starting mongod : [02:00:01.977] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55891,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55891 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 17593 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:00:00.768+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:00:00.768+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:00:00.768+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55891,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:00:00.769+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:00:00.771+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:00:00.771+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:00:00.801+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:00:00.801+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:00:00.801+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:00:00.845+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:00:00.845+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:00:01.376+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868001:376033][17593:0x7f876c1d2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:00:01.500+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868001:500189][17593:0x7f876c1d2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:00:01.637+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868001:637682][17593:0x7f876c1d2bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:00:01.777+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868001:777385][17593:0x7f876c1d2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:00:01.850+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868001:850589][17593:0x7f876c1d2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:00:01.921+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868001:921072][17593:0x7f876c1d2bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:00:01.921+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868001:921165][17593:0x7f876c1d2bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:00:01.922+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:00:01.977+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:00:01.977] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:55891 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:00:01.977] Error starting mongod : [02:00:01.977] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55891,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55891 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 17593 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:00:00.768+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:00:00.768+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:00:00.768+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55891,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:00:00.769+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:00:00.771+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:00:00.771+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:00:00.801+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:00:00.801+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:00:00.801+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:00:00.845+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:00:00.845+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:00:01.376+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868001:376033][17593:0x7f876c1d2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:00:01.500+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868001:500189][17593:0x7f876c1d2bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:00:01.977+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:00:01.977] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:00:01.977] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:55891 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:00:01.977] Error starting mongod : [02:00:01.977] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55891,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55891 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 17593 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:00:00.768+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:00:00.768+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:00:00.768+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55891,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:00:00.769+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:00:00.771+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:00:00.771+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:00:00.801+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:00:00.801+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:00:00.801+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:00:00.845+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:00:00.845+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:00:01.376+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868001:376033][17593:0x7f876c1d2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:00:01.500+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:00:01.977+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:00:01.977] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:00:01.977+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:00:01.977] Initiating stopEphemeralMongo [2021-10-22T02:00:01.977+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:00:01.977] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:00:01.978+0000] [.error] [src/director/director.go:executePlan:962] [02:00:01.978] Failed to apply action. Result = : [02:00:01.977] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:00:01.977] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:55891 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:00:01.977] Error starting mongod : [02:00:01.977] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55891,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55891 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 17593 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:00:00.768+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:00:00.768+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:00:00.768+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55891,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:00:00.769+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:00:00.771+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:00:00.771+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:00:00.801+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:00:00.801+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:00:00.801+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:00:00.845+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:00:00.845+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:00:01.376+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868001:376033][17593:0x7f876c1d2bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:00:01.978+0000] [.error] [src/director/director.go:planAndExecute:567] [02:00:01.978] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:00:01.978] Failed to apply action. Result = : [02:00:01.977] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:00:01.977] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:55891 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:00:01.977] Error starting mongod : [02:00:01.977] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55891,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55891 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 17593 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:00:00.768+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:00:00.768+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:00:00.768+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55891,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:00:00.769+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:00:00.771+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:00:00.771+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:00:00.801+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:00:00.801+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:00:00.801+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:00:00.845+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:00:00.845+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:00:01.376+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:00:01.978+0000] [.error] [src/director/director.go:mainLoop:399] [02:00:01.978] Failed to planAndExecute : [02:00:01.978] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:00:01.978] Failed to apply action. Result = : [02:00:01.977] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:00:01.977] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:55891 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:00:01.977] Error starting mongod : [02:00:01.977] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55891,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55891 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 17593 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:00:00.768+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:00:00.768+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:00:00.768+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55891,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:00:00.769+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:00:00.771+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:00:00.771+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:00:00.801+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:00:00.801+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:00:00.801+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:00:00.845+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:00:00.845+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:00:01.376+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:00:02.427+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:00:02.427] clusterConfig unchanged [2021-10-22T02:00:02.551+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:02.551] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:03.550+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:03.550] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:04.551+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:04.551] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:05.549+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:05.549] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:06.568+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:06.568] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:07.553+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:07.553] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:08.552+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:08.552] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:09.553+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:09.553] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:10.552+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:10.552] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:11.064+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:00:11.064] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.849578167s [2021-10-22T02:00:11.552+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:11.552] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:12.518+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:00:12.518] clusterConfig unchanged [2021-10-22T02:00:12.552+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:12.552] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:13.553+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:13.553] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:14.554+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:14.554] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:15.561+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:15.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:16.556+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:16.556] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:17.556+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:17.556] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:18.556+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:18.556] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:19.557+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:19.557] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:20.557+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:20.557] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:21.558+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:21.558] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:22.558+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:22.558] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:22.641+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:00:22.641] clusterConfig unchanged [2021-10-22T02:00:23.558+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:23.558] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:24.559+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:24.559] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:25.560+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:25.560] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:26.560+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:26.560] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:27.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:27.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:28.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:28.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:29.563+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:29.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:30.563+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:30.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:31.563+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:31.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:32.564+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:32.564] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:32.733+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:00:32.733] clusterConfig unchanged [2021-10-22T02:00:33.565+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:33.565] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:34.566+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:34.566] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:35.567+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:35.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:36.567+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:36.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:37.569+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:37.569] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:38.568+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:38.568] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:39.569+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:39.569] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:40.578+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:40.578] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:41.373+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:00:41.373] memory:rss=55492608 alloc=9699608 totalalloc=8561607648 mallocs=109151536 frees=109078090 heapinuse=16138240 heapobjects=73446 gcpausetotalns=1569973597 gcpauselastns=71608 gcnum=1291 utimens=175616130000 stimens=66541802000 minflt=452752 majflt=405 [2021-10-22T02:00:41.570+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:41.570] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:42.572+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:42.572] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:42.866+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:00:42.866] clusterConfig unchanged [2021-10-22T02:00:43.573+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:43.573] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:44.574+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:44.573] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:45.574+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:45.574] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:46.582+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:46.582] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:47.575+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:47.575] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:48.575+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:48.575] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:49.577+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:49.577] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:50.577+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:50.577] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:51.577+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:51.577] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:52.578+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:52.578] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:52.997+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:00:52.997] clusterConfig unchanged [2021-10-22T02:00:53.580+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:53.580] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:54.580+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:54.580] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:55.580+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:55.580] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:56.581+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:56.581] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:57.581+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:57.581] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:58.589+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:58.589] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:00:59.582+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:00:59.582] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:00.583+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:00.583] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:01.584+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:01.584] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:02.318+0000] [.info] [src/director/director.go:computePlan:280] [02:01:02.318] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:01:02.318+0000] [.info] [src/director/director.go:tracef:794] [02:01:02.318] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:01:02.318+0000] [.info] [src/director/director.go:tracef:794] [02:01:02.318] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:01:02.318+0000] [.info] [src/director/director.go:tracef:794] [02:01:02.318] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:01:02.318+0000] [.info] [src/director/director.go:tracef:794] [02:01:02.318] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:01:02.318+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:01:02.318] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:01:02.475+0000] [.info] [src/director/director.go:tracef:794] [02:01:02.475] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:01:02.475+0000] [.info] [src/director/director.go:tracef:794] [02:01:02.475] because [2021-10-22T02:01:02.475+0000] [.info] [src/director/director.go:tracef:794] [02:01:02.475] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:01:02.475+0000] [.info] [src/director/director.go:tracef:794] [02:01:02.475] because ['currentState.CanBounceStop' = true] [2021-10-22T02:01:02.586+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:02.586] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:02.628+0000] [.info] [src/director/director.go:tracef:794] [02:01:02.628] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:01:02.628+0000] [.info] [src/director/director.go:tracef:794] [02:01:02.628] because [2021-10-22T02:01:02.628+0000] [.info] [src/director/director.go:tracef:794] [02:01:02.628] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:01:02.628+0000] [.info] [src/director/director.go:tracef:794] [02:01:02.628] because ['currentState.Up' = false] [2021-10-22T02:01:02.628+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:01:02.628] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:01:02.628+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:01:02.628] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:01:02.705+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:01:02.705] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:01:02.714+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:01:02.714] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:01:02.714+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:01:02.714] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:01:02.715+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:01:02.715] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:01:02.716+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:01:02.716] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:01:02.716+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:01:02.716] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:01:02.716+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:01:02.716] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:01:02.718+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:01:02.718] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:01:02.957+0000] [.info] [src/director/director.go:tracef:794] [02:01:02.957] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:01:02.957+0000] [.info] [src/director/director.go:tracef:794] [02:01:02.957] because ['currentState.Up' = false] [2021-10-22T02:01:03.178+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:01:03.178] clusterConfig unchanged [2021-10-22T02:01:03.189+0000] [.info] [src/director/director.go:tracef:794] [02:01:03.189] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:01:03.189+0000] [.info] [src/director/director.go:tracef:794] [02:01:03.189] because ['currentState.Up' = false] [2021-10-22T02:01:03.189+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:01:03.189] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:01:03.399+0000] [.info] [src/director/director.go:tracef:794] [02:01:03.399] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:01:03.399+0000] [.info] [src/director/director.go:tracef:794] [02:01:03.399] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:01:03.587+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:03.587] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:03.610+0000] [.info] [src/director/director.go:tracef:794] [02:01:03.610] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:01:03.610+0000] [.info] [src/director/director.go:tracef:794] [02:01:03.610] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:01:03.610+0000] [.info] [src/director/director.go:tracef:794] [02:01:03.610] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:01:03.610+0000] [.info] [src/director/director.go:tracef:794] [02:01:03.610] because ['currentState.Up' = false] [2021-10-22T02:01:03.610+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:01:03.610] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:01:03.610+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:01:03.610] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:01:03.645+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:01:03.645] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:01:03.646+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:01:03.646] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38404 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:01:04.310+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:01:04.310] clusterConfig unchanged [2021-10-22T02:01:04.401+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:01:04.401] ...Finished start [pid=36854] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38404,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:01:04.558+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:01:04.558] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 156.117261ms. [2021-10-22T02:01:04.558+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:01:04.558] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38404 (local=false) [2021-10-22T02:01:04.558+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:01:04.558] Dropping existing documents from system.replset [2021-10-22T02:01:04.589+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:04.589] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:04.715+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:01:04.715] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:01:04.730+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:01:04.730] Inserting oplogTruncateAfterPoint document [2021-10-22T02:01:04.730+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:01:04.730] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:01:04.731+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:01:04.731] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:01:04.731+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:01:04.731] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:01:04.744+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:01:04.744] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:01:04.744+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:01:04.744] Setting the minvalid document to [{_id ObjectID("61721b60523370002becc72a")} {t -1} {ts {0 1}}] [2021-10-22T02:01:04.757+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:01:04.757] Inserted [{_id ObjectID("61721b60523370002becc72a")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:01:04.757+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:01:04.757] Inserting new replica set configuration document. [2021-10-22T02:01:04.758+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:01:04.757] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:01:04.770+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:01:04.770] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:01:04.784+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:01:04.784] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:01:04.784+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:01:04.784] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:01:04.784+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:01:04.784] Initiating stopEphemeralMongo [2021-10-22T02:01:04.784+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:01:04.784] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38404 (local=false) [2021-10-22T02:01:04.784+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:01:04.784] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38404 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:01:05.237+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:01:05.237] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.091303ms. [2021-10-22T02:01:05.237+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:01:05.237] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38404 (local=false) [2021-10-22T02:01:05.237+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:01:05.237] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36µs. [2021-10-22T02:01:05.238+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:01:05.238] Successfully finished running stopEphemeralMongo [2021-10-22T02:01:05.238+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:01:05.238] Successfully stopped existing ephemeral mongod. [2021-10-22T02:01:05.238+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:01:05.238] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:01:05.238+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:01:05.238] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:01:05.275+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:01:05.275] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:01:05.275+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:01:05.275] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42766 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:01:05.433+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:01:05.433] clusterConfig unchanged [2021-10-22T02:01:05.590+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:05.590] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:06.458+0000] [.error] [src/action/start.go:func1:93] [103] [02:01:06.458] Error starting mongod : [02:01:06.457] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42766,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42766 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 38519 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:01:05.338+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:01:05.338+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:01:05.338+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42766,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:01:05.339+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:01:05.341+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:01:05.341+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:01:05.359+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:01:05.359+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:01:05.359+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:01:05.406+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:01:05.406+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:01:05.933+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868065:933247][38519:0x7f1e981a6bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:01:06.041+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868066:41557][38519:0x7f1e981a6bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:01:06.156+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868066:156005][38519:0x7f1e981a6bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:01:06.272+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868066:272074][38519:0x7f1e981a6bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:01:06.344+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868066:344477][38519:0x7f1e981a6bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:01:06.408+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868066:408104][38519:0x7f1e981a6bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:01:06.408+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868066:408177][38519:0x7f1e981a6bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:01:06.409+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:01:06.459+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:01:06.459] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42766 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:01:06.458] Error starting mongod : [02:01:06.457] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42766,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42766 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 38519 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:01:05.338+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:01:05.338+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:01:05.338+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42766,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:01:05.339+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:01:05.341+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:01:05.341+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:01:05.359+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:01:05.359+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:01:05.359+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:01:05.406+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:01:05.406+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:01:05.933+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868065:933247][38519:0x7f1e981a6bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:01:06.041+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868066:41557][38519:0x7f1e981a6bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:01:06.460+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:01:06.460] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:01:06.459] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42766 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:01:06.458] Error starting mongod : [02:01:06.457] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42766,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42766 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 38519 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:01:05.338+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:01:05.338+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:01:05.338+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42766,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:01:05.339+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:01:05.341+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:01:05.341+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:01:05.359+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:01:05.359+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:01:05.359+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:01:05.406+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:01:05.406+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:01:05.933+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868065:933247][38519:0x7f1e981a6bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:01:06.041+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ct... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:01:06.461+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:01:06.460] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:01:06.461+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:01:06.461] Initiating stopEphemeralMongo [2021-10-22T02:01:06.461+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:01:06.461] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:01:06.468+0000] [.error] [src/director/director.go:executePlan:962] [02:01:06.468] Failed to apply action. Result = : [02:01:06.460] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:01:06.459] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42766 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:01:06.458] Error starting mongod : [02:01:06.457] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42766,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42766 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 38519 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:01:05.338+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:01:05.338+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:01:05.338+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42766,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:01:05.339+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:01:05.341+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:01:05.341+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:01:05.359+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:01:05.359+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:01:05.359+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:01:05.406+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:01:05.406+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:01:05.933+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868065:933247][38519:0x7f1e981a6bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:01:06.468+0000] [.error] [src/director/director.go:planAndExecute:567] [02:01:06.468] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:01:06.468] Failed to apply action. Result = : [02:01:06.460] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:01:06.459] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42766 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:01:06.458] Error starting mongod : [02:01:06.457] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42766,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42766 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 38519 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:01:05.338+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:01:05.338+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:01:05.338+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42766,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:01:05.339+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:01:05.341+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:01:05.341+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:01:05.359+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:01:05.359+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:01:05.359+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:01:05.406+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:01:05.406+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:01:05.933+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:01:06.468+0000] [.error] [src/director/director.go:mainLoop:399] [02:01:06.468] Failed to planAndExecute : [02:01:06.468] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:01:06.468] Failed to apply action. Result = : [02:01:06.460] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:01:06.459] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42766 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:01:06.458] Error starting mongod : [02:01:06.457] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42766,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42766 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 38519 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:01:05.338+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:01:05.338+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:01:05.338+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42766,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:01:05.339+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:01:05.341+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:01:05.341+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:01:05.359+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:01:05.359+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:01:05.359+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:01:05.406+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:01:05.406+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:01:05.933+00:00"},"s":"I", "c":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:01:06.535+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:01:06.535] clusterConfig unchanged [2021-10-22T02:01:06.590+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:06.590] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:07.590+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:07.590] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:07.776+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:01:07.776] clusterConfig unchanged [2021-10-22T02:01:08.590+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:08.590] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:09.590+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:09.590] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:10.591+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:10.591] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:11.590+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:11.590] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:12.593+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:12.593] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:13.594+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:13.594] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:14.593+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:14.593] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:15.316+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:01:15.316] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.597223883s [2021-10-22T02:01:15.599+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:15.599] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:16.593+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:16.593] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:17.594+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:17.594] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:17.898+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:01:17.898] clusterConfig unchanged [2021-10-22T02:01:18.563+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:01:18.563] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:01:18.565+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:18.565] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:01:18.565+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [02:01:18.565] Could not get version : [02:01:18.565] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:01:18.565+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [02:01:18.565] Error getting version for log lines. Skipping : [02:01:18.565] Could not get version : [02:01:18.565] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:01:18.594+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:18.594] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:19.596+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:19.596] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:20.596+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:20.596] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:21.596+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:21.596] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:22.597+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:22.597] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:23.597+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:23.597] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:24.599+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:24.599] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:25.607+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:25.607] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:26.599+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:26.599] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:27.600+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:27.600] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:28.009+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:01:28.009] clusterConfig unchanged [2021-10-22T02:01:28.600+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:28.600] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:29.600+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:29.600] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:30.601+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:30.601] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:31.602+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:31.602] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:32.602+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:32.602] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:33.603+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:33.603] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:34.604+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:34.604] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:35.605+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:35.605] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:36.607+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:36.607] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:37.606+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:37.606] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:38.125+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:01:38.125] clusterConfig unchanged [2021-10-22T02:01:38.606+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:38.606] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:39.606+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:39.606] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:40.607+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:40.607] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:41.374+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:01:41.374] memory:rss=54407168 alloc=10350656 totalalloc=8652192776 mallocs=110307864 frees=110226191 heapinuse=16302080 heapobjects=81673 gcpausetotalns=1572352302 gcpauselastns=77980 gcnum=1303 utimens=177859642000 stimens=67532423000 minflt=457950 majflt=405 [2021-10-22T02:01:41.607+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:41.607] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:42.608+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:42.608] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:43.608+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:43.608] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:44.609+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:44.609] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:45.617+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:45.617] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:46.610+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:46.610] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:47.611+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:47.611] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:48.218+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:01:48.218] clusterConfig unchanged [2021-10-22T02:01:48.611+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:48.611] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:49.611+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:49.611] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:50.612+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:50.612] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:51.613+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:51.613] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:52.616+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:52.616] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:53.614+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:53.614] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:54.614+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:54.614] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:55.615+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:55.615] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:56.616+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:56.616] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:57.617+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:57.617] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:58.320+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:01:58.320] clusterConfig unchanged [2021-10-22T02:01:58.617+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:58.617] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:01:59.618+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:01:59.618] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:00.619+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:00.619] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:01.620+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:01.620] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:02.621+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:02.621] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:03.621+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:03.621] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:04.622+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:04.622] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:05.624+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:05.624] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:06.625+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:06.625] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:06.645+0000] [.info] [src/director/director.go:computePlan:280] [02:02:06.645] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:02:06.646+0000] [.info] [src/director/director.go:tracef:794] [02:02:06.646] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:02:06.646+0000] [.info] [src/director/director.go:tracef:794] [02:02:06.646] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:02:06.646+0000] [.info] [src/director/director.go:tracef:794] [02:02:06.646] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:02:06.646+0000] [.info] [src/director/director.go:tracef:794] [02:02:06.646] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:02:06.646+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:02:06.646] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:02:06.805+0000] [.info] [src/director/director.go:tracef:794] [02:02:06.805] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:02:06.805+0000] [.info] [src/director/director.go:tracef:794] [02:02:06.805] because [2021-10-22T02:02:06.805+0000] [.info] [src/director/director.go:tracef:794] [02:02:06.805] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:02:06.805+0000] [.info] [src/director/director.go:tracef:794] [02:02:06.805] because ['currentState.CanBounceStop' = true] [2021-10-22T02:02:06.967+0000] [.info] [src/director/director.go:tracef:794] [02:02:06.967] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:02:06.967+0000] [.info] [src/director/director.go:tracef:794] [02:02:06.967] because [2021-10-22T02:02:06.967+0000] [.info] [src/director/director.go:tracef:794] [02:02:06.967] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:02:06.967+0000] [.info] [src/director/director.go:tracef:794] [02:02:06.967] because ['currentState.Up' = false] [2021-10-22T02:02:06.967+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:02:06.967] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:02:06.967+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:02:06.967] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:02:07.054+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:02:07.054] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:02:07.062+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:02:07.062] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:02:07.062+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:02:07.062] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:02:07.063+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:02:07.063] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:02:07.064+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:02:07.064] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:02:07.064+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:02:07.064] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:02:07.064+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:02:07.064] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:02:07.065+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:02:07.065] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:02:07.292+0000] [.info] [src/director/director.go:tracef:794] [02:02:07.292] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:02:07.292+0000] [.info] [src/director/director.go:tracef:794] [02:02:07.292] because ['currentState.Up' = false] [2021-10-22T02:02:07.489+0000] [.info] [src/director/director.go:tracef:794] [02:02:07.489] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:02:07.489+0000] [.info] [src/director/director.go:tracef:794] [02:02:07.489] because ['currentState.Up' = false] [2021-10-22T02:02:07.489+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:02:07.489] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:02:07.634+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:07.634] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:07.713+0000] [.info] [src/director/director.go:tracef:794] [02:02:07.713] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:02:07.713+0000] [.info] [src/director/director.go:tracef:794] [02:02:07.713] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:02:07.939+0000] [.info] [src/director/director.go:tracef:794] [02:02:07.939] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:02:07.939+0000] [.info] [src/director/director.go:tracef:794] [02:02:07.939] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:02:07.940+0000] [.info] [src/director/director.go:tracef:794] [02:02:07.939] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:02:07.940+0000] [.info] [src/director/director.go:tracef:794] [02:02:07.940] because ['currentState.Up' = false] [2021-10-22T02:02:07.940+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:02:07.940] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:02:07.940+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:02:07.940] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:02:07.974+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:02:07.974] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:02:07.974+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:02:07.974] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57824 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:02:08.531+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:02:08.531] clusterConfig unchanged [2021-10-22T02:02:08.633+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:08.633] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:08.719+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:02:08.719] ...Finished start [pid=57582] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57824,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:02:08.839+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:02:08.839] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 119.404116ms. [2021-10-22T02:02:08.839+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:02:08.839] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57824 (local=false) [2021-10-22T02:02:08.839+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:02:08.839] Dropping existing documents from system.replset [2021-10-22T02:02:08.970+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:02:08.970] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:02:08.974+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:02:08.974] Inserting oplogTruncateAfterPoint document [2021-10-22T02:02:08.974+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:02:08.974] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:02:08.975+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:02:08.975] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:02:08.975+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:02:08.975] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:02:08.988+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:02:08.988] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:02:08.988+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:02:08.988] Setting the minvalid document to [{_id ObjectID("61721ba0523370002becc738")} {t -1} {ts {0 1}}] [2021-10-22T02:02:09.000+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:02:09.000] Inserted [{_id ObjectID("61721ba0523370002becc738")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:02:09.000+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:02:09.000] Inserting new replica set configuration document. [2021-10-22T02:02:09.000+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:02:09.000] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:02:09.013+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:02:09.013] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:02:09.017+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:02:09.017] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:02:09.017+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:02:09.017] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:02:09.017+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:02:09.017] Initiating stopEphemeralMongo [2021-10-22T02:02:09.017+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:02:09.017] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57824 (local=false) [2021-10-22T02:02:09.017+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:02:09.017] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57824 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:02:09.384+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:02:09.384] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.626292ms. [2021-10-22T02:02:09.384+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:02:09.384] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57824 (local=false) [2021-10-22T02:02:09.384+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:02:09.384] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.456µs. [2021-10-22T02:02:09.384+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:02:09.384] Successfully finished running stopEphemeralMongo [2021-10-22T02:02:09.384+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:02:09.384] Successfully stopped existing ephemeral mongod. [2021-10-22T02:02:09.384+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:02:09.384] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:02:09.384+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:02:09.384] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:02:09.418+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:02:09.418] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:02:09.419+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:02:09.419] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 54873 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:02:09.636+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:09.636] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:09.643+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:02:09.643] clusterConfig unchanged [2021-10-22T02:02:10.646+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:10.646] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:10.732+0000] [.error] [src/action/start.go:func1:93] [103] [02:02:10.732] Error starting mongod : [02:02:10.732] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 54873 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:02:09.450Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 59064 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:02:09.469+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:02:09.469+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:02:09.469+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:02:09.471+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:02:09.473+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:02:09.473+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:02:09.483+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:02:09.483+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:02:09.484+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:02:09.529+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:02:09.529+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:02:10.127+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868130:126981][59064:0x7f7d0c250bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:02:10.263+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868130:263391][59064:0x7f7d0c250bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:02:10.376+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868130:376122][59064:0x7f7d0c250bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T02:02:10.531+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868130:531166][59064:0x7f7d0c250bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:02:10.609+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868130:608983][59064:0x7f7d0c250bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:02:10.680+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868130:680628][59064:0x7f7d0c250bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:02:10.680+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868130:680687][59064:0x7f7d0c250bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:02:10.733+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:02:10.733] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:54873 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:02:10.732] Error starting mongod : [02:02:10.732] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 54873 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:02:09.450Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 59064 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:02:09.469+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:02:09.469+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:02:09.469+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:02:09.471+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:02:09.473+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:02:09.473+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:02:09.483+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:02:09.483+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:02:09.484+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:02:09.529+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:02:09.529+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:02:10.127+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868130:126981][59064:0x7f7d0c250bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:02:10.263+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:02:10.733+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:02:10.733] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:02:10.733] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:54873 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:02:10.732] Error starting mongod : [02:02:10.732] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 54873 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:02:09.450Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 59064 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:02:09.469+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:02:09.469+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:02:09.469+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:02:09.471+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:02:09.473+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:02:09.473+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:02:09.483+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:02:09.483+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:02:09.484+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:02:09.529+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:02:09.529+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:02:10.127+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868130:126981][59064:0x7f7d0c250bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:02:10.733+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:02:10.733] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:02:10.733+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:02:10.733] Initiating stopEphemeralMongo [2021-10-22T02:02:10.733+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:02:10.733] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:02:10.734+0000] [.error] [src/director/director.go:executePlan:962] [02:02:10.734] Failed to apply action. Result = : [02:02:10.733] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:02:10.733] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:54873 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:02:10.732] Error starting mongod : [02:02:10.732] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 54873 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:02:09.450Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 59064 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:02:09.469+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:02:09.469+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:02:09.469+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:02:09.471+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:02:09.473+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:02:09.473+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:02:09.483+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:02:09.483+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:02:09.484+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:02:09.529+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:02:09.529+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:02:10.127+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:02:10.734+0000] [.error] [src/director/director.go:planAndExecute:567] [02:02:10.734] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:02:10.734] Failed to apply action. Result = : [02:02:10.733] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:02:10.733] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:54873 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:02:10.732] Error starting mongod : [02:02:10.732] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 54873 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:02:09.450Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 59064 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:02:09.469+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:02:09.469+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:02:09.469+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:02:09.471+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:02:09.473+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:02:09.473+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:02:09.483+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:02:09.483+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:02:09.484+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:02:09.529+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:02:09.529+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:02:10.734+0000] [.error] [src/director/director.go:mainLoop:399] [02:02:10.734] Failed to planAndExecute : [02:02:10.734] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:02:10.734] Failed to apply action. Result = : [02:02:10.733] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:02:10.733] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:54873 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:02:10.732] Error starting mongod : [02:02:10.732] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 54873 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:02:09.450Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 59064 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:02:09.469+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:02:09.469+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:02:09.469+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54873,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:02:09.471+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:02:09.473+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:02:09.473+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:02:09.483+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:02:09.483+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:02:09.484+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:02:09.529+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:02:09.529+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:02:10.756+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:02:10.756] clusterConfig unchanged [2021-10-22T02:02:11.647+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:11.647] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:11.976+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:02:11.976] clusterConfig unchanged [2021-10-22T02:02:12.646+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:12.646] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:13.648+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:13.648] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:14.648+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:14.648] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:15.647+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:15.647] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:16.657+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:16.657] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:17.650+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:17.650] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:18.650+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:18.650] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:19.650+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:19.650] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:19.723+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:02:19.723] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.657143026s [2021-10-22T02:02:20.649+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:20.649] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:21.656+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:21.656] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:22.079+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:02:22.079] clusterConfig unchanged [2021-10-22T02:02:22.650+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:22.650] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:23.651+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:23.651] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:24.651+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:24.651] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:25.653+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:25.653] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:26.653+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:26.653] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:27.568+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:02:27.568] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:02:27.569+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:02:27.569] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:02:27.654+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:27.654] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:28.654+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:28.654] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:29.655+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:29.655] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:30.655+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:30.655] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:31.657+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:31.657] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:32.203+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:02:32.203] clusterConfig unchanged [2021-10-22T02:02:32.660+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:32.660] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:33.665+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:33.665] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:34.659+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:34.659] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:35.660+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:35.660] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:36.660+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:36.660] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:37.662+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [02:02:37.662] lsblk cache invalidated, running lsblk command [2021-10-22T02:02:37.663+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:37.663] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:38.661+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:38.661] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:39.662+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:39.662] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:40.663+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:40.663] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:41.374+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:02:41.374] memory:rss=55721984 alloc=12543680 totalalloc=8743014464 mallocs=111470857 frees=111365714 heapinuse=17252352 heapobjects=105143 gcpausetotalns=1574717837 gcpauselastns=151857 gcnum=1315 utimens=180090772000 stimens=68475241000 minflt=463103 majflt=405 [2021-10-22T02:02:41.663+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:41.663] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:42.343+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:02:42.343] clusterConfig unchanged [2021-10-22T02:02:42.671+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:42.671] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:43.664+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:43.664] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:44.364+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [02:02:44.364] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:54873 (local=false) because it is not up [2021-10-22T02:02:44.664+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:44.664] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:45.664+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:45.664] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:46.665+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:46.665] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:47.665+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:47.665] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:48.667+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:48.667] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:49.667+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:49.667] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:50.669+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:50.669] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:51.669+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:51.669] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:52.483+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:02:52.483] clusterConfig unchanged [2021-10-22T02:02:52.671+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:52.671] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:53.670+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:53.670] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:54.670+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:54.670] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:55.671+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:55.671] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:56.671+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:56.671] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:57.672+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:57.672] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:58.672+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:58.672] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:02:59.673+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:02:59.673] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:00.673+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:00.673] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:01.675+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:01.675] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:02.586+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:03:02.586] clusterConfig unchanged [2021-10-22T02:03:02.675+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:02.675] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:03.675+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:03.675] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:04.678+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:04.678] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:05.675+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:05.675] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:06.677+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:06.677] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:07.679+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:07.679] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:08.677+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:08.677] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:09.678+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:09.678] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:10.678+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:10.678] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:10.903+0000] [.info] [src/director/director.go:computePlan:280] [02:03:10.903] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:03:10.903+0000] [.info] [src/director/director.go:tracef:794] [02:03:10.903] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:03:10.903+0000] [.info] [src/director/director.go:tracef:794] [02:03:10.903] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:03:10.903+0000] [.info] [src/director/director.go:tracef:794] [02:03:10.903] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:03:10.903+0000] [.info] [src/director/director.go:tracef:794] [02:03:10.903] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:03:10.903+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:03:10.903] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:03:11.067+0000] [.info] [src/director/director.go:tracef:794] [02:03:11.067] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:03:11.067+0000] [.info] [src/director/director.go:tracef:794] [02:03:11.067] because [2021-10-22T02:03:11.067+0000] [.info] [src/director/director.go:tracef:794] [02:03:11.067] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:03:11.067+0000] [.info] [src/director/director.go:tracef:794] [02:03:11.067] because ['currentState.CanBounceStop' = true] [2021-10-22T02:03:11.230+0000] [.info] [src/director/director.go:tracef:794] [02:03:11.230] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:03:11.230+0000] [.info] [src/director/director.go:tracef:794] [02:03:11.230] because [2021-10-22T02:03:11.231+0000] [.info] [src/director/director.go:tracef:794] [02:03:11.230] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:03:11.231+0000] [.info] [src/director/director.go:tracef:794] [02:03:11.231] because ['currentState.Up' = false] [2021-10-22T02:03:11.231+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:03:11.231] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:03:11.231+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:03:11.231] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:03:11.309+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:03:11.309] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:03:11.310+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:03:11.310] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:03:11.318+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:03:11.318] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:03:11.319+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:03:11.319] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:03:11.320+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:03:11.320] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:03:11.320+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:03:11.320] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:03:11.320+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:03:11.320] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:03:11.322+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:03:11.322] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:03:11.544+0000] [.info] [src/director/director.go:tracef:794] [02:03:11.544] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:03:11.544+0000] [.info] [src/director/director.go:tracef:794] [02:03:11.544] because ['currentState.Up' = false] [2021-10-22T02:03:11.680+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:11.680] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:11.767+0000] [.info] [src/director/director.go:tracef:794] [02:03:11.767] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:03:11.767+0000] [.info] [src/director/director.go:tracef:794] [02:03:11.767] because ['currentState.Up' = false] [2021-10-22T02:03:11.767+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:03:11.767] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:03:12.016+0000] [.info] [src/director/director.go:tracef:794] [02:03:12.016] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:03:12.016+0000] [.info] [src/director/director.go:tracef:794] [02:03:12.016] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:03:12.226+0000] [.info] [src/director/director.go:tracef:794] [02:03:12.226] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:03:12.226+0000] [.info] [src/director/director.go:tracef:794] [02:03:12.226] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:03:12.226+0000] [.info] [src/director/director.go:tracef:794] [02:03:12.226] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:03:12.226+0000] [.info] [src/director/director.go:tracef:794] [02:03:12.226] because ['currentState.Up' = false] [2021-10-22T02:03:12.227+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:03:12.226] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:03:12.227+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:03:12.227] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:03:12.264+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:03:12.264] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:03:12.265+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:03:12.265] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36455 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:03:12.704+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:12.704] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:12.777+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:03:12.777] clusterConfig unchanged [2021-10-22T02:03:13.009+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:03:13.009] ...Finished start [pid=4358] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36455,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:03:13.230+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:03:13.230] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 221.373862ms. [2021-10-22T02:03:13.230+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:03:13.230] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36455 (local=false) [2021-10-22T02:03:13.230+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:03:13.230] Dropping existing documents from system.replset [2021-10-22T02:03:13.376+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:03:13.376] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:03:13.380+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:03:13.380] Inserting oplogTruncateAfterPoint document [2021-10-22T02:03:13.380+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:03:13.380] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:03:13.381+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:03:13.381] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:03:13.381+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:03:13.381] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:03:13.394+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:03:13.394] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:03:13.394+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:03:13.394] Setting the minvalid document to [{_id ObjectID("61721be1523370002becc746")} {t -1} {ts {0 1}}] [2021-10-22T02:03:13.406+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:03:13.406] Inserted [{_id ObjectID("61721be1523370002becc746")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:03:13.407+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:03:13.406] Inserting new replica set configuration document. [2021-10-22T02:03:13.407+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:03:13.407] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:03:13.419+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:03:13.418] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:03:13.422+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:03:13.422] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:03:13.422+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:03:13.422] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:03:13.422+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:03:13.422] Initiating stopEphemeralMongo [2021-10-22T02:03:13.422+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:03:13.422] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36455 (local=false) [2021-10-22T02:03:13.422+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:03:13.422] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36455 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:03:13.697+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:13.697] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:13.854+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:03:13.853] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.688632ms. [2021-10-22T02:03:13.854+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:03:13.854] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36455 (local=false) [2021-10-22T02:03:13.854+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:03:13.854] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 25.677µs. [2021-10-22T02:03:13.854+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:03:13.854] Successfully finished running stopEphemeralMongo [2021-10-22T02:03:13.854+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:03:13.854] Successfully stopped existing ephemeral mongod. [2021-10-22T02:03:13.854+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:03:13.854] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:03:13.854+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:03:13.854] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:03:13.886+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:03:13.886] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:03:13.887+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:03:13.887] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49255 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:03:13.916+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:03:13.916] clusterConfig unchanged [2021-10-22T02:03:14.701+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:14.701] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:15.058+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:03:15.058] clusterConfig unchanged [2021-10-22T02:03:15.192+0000] [.error] [src/action/start.go:func1:93] [103] [02:03:15.192] Error starting mongod : [02:03:15.192] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49255,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49255 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 6002 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:03:13.973+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:03:13.973+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:03:13.973+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49255,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:03:13.975+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:03:13.977+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:03:13.978+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:03:13.997+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:03:13.997+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:03:13.997+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:03:14.043+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:03:14.043+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:03:14.611+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868194:611358][6002:0x7f6cd0e44bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:03:14.758+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868194:758776][6002:0x7f6cd0e44bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:03:14.858+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868194:858249][6002:0x7f6cd0e44bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:03:14.973+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868194:973605][6002:0x7f6cd0e44bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:03:15.068+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868195:68684][6002:0x7f6cd0e44bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:03:15.143+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868195:143848][6002:0x7f6cd0e44bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:03:15.143+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868195:143919][6002:0x7f6cd0e44bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:03:15.145+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:03:15.192+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:03:15.192] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49255 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:03:15.192] Error starting mongod : [02:03:15.192] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49255,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49255 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 6002 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:03:13.973+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:03:13.973+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:03:13.973+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49255,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:03:13.975+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:03:13.977+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:03:13.978+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:03:13.997+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:03:13.997+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:03:13.997+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:03:14.043+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:03:14.043+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:03:14.611+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868194:611358][6002:0x7f6cd0e44bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:03:14.758+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868194:758776][6002:0x7f6cd0e44bc0], txn-recove...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:03:15.192+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:03:15.192] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:03:15.192] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49255 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:03:15.192] Error starting mongod : [02:03:15.192] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49255,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49255 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 6002 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:03:13.973+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:03:13.973+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:03:13.973+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49255,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:03:13.975+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:03:13.977+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:03:13.978+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:03:13.997+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:03:13.997+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:03:13.997+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:03:14.043+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:03:14.043+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:03:14.611+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868194:611358][6002:0x7f6cd0e44bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:03:14.758+00:00"},"s":"I", "c":"STORAGE", "id":22430,... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:03:15.192+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:03:15.192] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:03:15.192+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:03:15.192] Initiating stopEphemeralMongo [2021-10-22T02:03:15.193+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:03:15.192] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:03:15.193+0000] [.error] [src/director/director.go:executePlan:962] [02:03:15.193] Failed to apply action. Result = : [02:03:15.192] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:03:15.192] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49255 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:03:15.192] Error starting mongod : [02:03:15.192] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49255,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49255 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 6002 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:03:13.973+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:03:13.973+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:03:13.973+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49255,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:03:13.975+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:03:13.977+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:03:13.978+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:03:13.997+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:03:13.997+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:03:13.997+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:03:14.043+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:03:14.043+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:03:14.611+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868194:611358][6002:0x7f6cd0e44bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:03:15.193+0000] [.error] [src/director/director.go:planAndExecute:567] [02:03:15.193] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:03:15.193] Failed to apply action. Result = : [02:03:15.192] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:03:15.192] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49255 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:03:15.192] Error starting mongod : [02:03:15.192] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49255,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49255 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 6002 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:03:13.973+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:03:13.973+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:03:13.973+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49255,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:03:13.975+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:03:13.977+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:03:13.978+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:03:13.997+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:03:13.997+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:03:13.997+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:03:14.043+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:03:14.043+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:03:14.611+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger ... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:03:15.193+0000] [.error] [src/director/director.go:mainLoop:399] [02:03:15.193] Failed to planAndExecute : [02:03:15.193] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:03:15.193] Failed to apply action. Result = : [02:03:15.192] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:03:15.192] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49255 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:03:15.192] Error starting mongod : [02:03:15.192] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49255,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49255 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 6002 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:03:13.973+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:03:13.973+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:03:13.973+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49255,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:03:13.975+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:03:13.977+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:03:13.978+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:03:13.997+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:03:13.997+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:03:13.997+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:03:14.043+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:03:14.043+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:03:14.611+00:00"},"s":"I"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:03:15.686+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:15.686] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:16.281+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:03:16.281] clusterConfig unchanged [2021-10-22T02:03:16.686+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:16.686] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:17.686+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:17.686] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:18.686+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:18.686] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:19.688+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:19.688] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:20.687+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:20.687] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:21.686+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:21.686] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:22.687+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:22.687] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:23.690+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:23.690] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:24.206+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:03:24.206] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.883321811s [2021-10-22T02:03:24.690+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:24.690] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:25.690+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:25.690] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:26.384+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:03:26.384] clusterConfig unchanged [2021-10-22T02:03:26.693+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:26.693] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:27.690+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:27.690] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:28.693+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:28.693] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:29.699+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:29.699] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:30.694+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:30.694] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:31.693+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:31.693] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:32.697+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:32.697] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:33.697+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:33.697] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:34.698+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:34.698] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:35.697+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:35.696] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:36.505+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:03:36.505] clusterConfig unchanged [2021-10-22T02:03:36.700+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:36.700] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:37.697+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:37.697] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:38.700+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:38.700] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:39.702+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:39.702] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:40.702+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:40.702] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:41.375+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:03:41.375] memory:rss=56696832 alloc=15226560 totalalloc=8835155472 mallocs=112645662 frees=112510069 heapinuse=18636800 heapobjects=135593 gcpausetotalns=1585583455 gcpauselastns=91984 gcnum=1327 utimens=182944836000 stimens=69860499000 minflt=468878 majflt=405 [2021-10-22T02:03:41.710+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:41.710] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:42.702+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:42.702] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:43.702+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:43.702] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:44.705+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:44.705] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:45.703+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:45.703] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:46.648+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:03:46.647] clusterConfig unchanged [2021-10-22T02:03:46.705+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:46.705] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:47.708+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:47.708] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:48.705+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:48.705] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:49.706+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:49.706] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:50.706+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:50.706] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:51.714+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:51.714] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:52.707+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:52.707] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:53.708+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:53.708] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:54.709+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:54.709] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:55.709+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:55.709] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:56.710+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:56.710] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:56.779+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:03:56.779] clusterConfig unchanged [2021-10-22T02:03:57.711+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:57.711] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:58.711+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:58.711] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:03:59.712+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:03:59.712] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:00.712+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:00.712] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:01.713+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:01.713] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:02.713+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:02.713] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:03.714+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:03.714] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:04.715+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:04.715] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:05.716+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:05.716] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:06.716+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:06.716] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:06.880+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:04:06.880] clusterConfig unchanged [2021-10-22T02:04:07.717+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:07.717] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:08.717+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:08.717] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:09.718+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:09.718] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:10.718+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:10.718] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:11.719+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:11.719] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:12.720+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:12.720] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:13.726+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:13.726] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:14.722+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:14.722] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:15.541+0000] [.info] [src/director/director.go:computePlan:280] [02:04:15.541] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:04:15.541+0000] [.info] [src/director/director.go:tracef:794] [02:04:15.541] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:04:15.541+0000] [.info] [src/director/director.go:tracef:794] [02:04:15.541] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:04:15.541+0000] [.info] [src/director/director.go:tracef:794] [02:04:15.541] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:04:15.541+0000] [.info] [src/director/director.go:tracef:794] [02:04:15.541] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:04:15.542+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:04:15.542] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:04:15.733+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:15.733] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:15.825+0000] [.info] [src/director/director.go:tracef:794] [02:04:15.825] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:04:15.825+0000] [.info] [src/director/director.go:tracef:794] [02:04:15.825] because [2021-10-22T02:04:15.825+0000] [.info] [src/director/director.go:tracef:794] [02:04:15.825] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:04:15.825+0000] [.info] [src/director/director.go:tracef:794] [02:04:15.825] because ['currentState.CanBounceStop' = true] [2021-10-22T02:04:15.994+0000] [.info] [src/director/director.go:tracef:794] [02:04:15.994] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:04:15.994+0000] [.info] [src/director/director.go:tracef:794] [02:04:15.994] because [2021-10-22T02:04:15.994+0000] [.info] [src/director/director.go:tracef:794] [02:04:15.994] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:04:15.994+0000] [.info] [src/director/director.go:tracef:794] [02:04:15.994] because ['currentState.Up' = false] [2021-10-22T02:04:15.994+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:04:15.994] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:04:15.994+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:04:15.994] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:04:16.087+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:04:16.087] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:04:16.095+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:04:16.095] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:04:16.096+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:04:16.096] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:04:16.096+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:04:16.096] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:04:16.097+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:04:16.097] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:04:16.097+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:04:16.097] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:04:16.097+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:04:16.097] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:04:16.099+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:04:16.099] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:04:16.338+0000] [.info] [src/director/director.go:tracef:794] [02:04:16.338] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:04:16.338+0000] [.info] [src/director/director.go:tracef:794] [02:04:16.338] because ['currentState.Up' = false] [2021-10-22T02:04:16.553+0000] [.info] [src/director/director.go:tracef:794] [02:04:16.553] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:04:16.554+0000] [.info] [src/director/director.go:tracef:794] [02:04:16.554] because ['currentState.Up' = false] [2021-10-22T02:04:16.554+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:04:16.554] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:04:16.725+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:16.725] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:16.781+0000] [.info] [src/director/director.go:tracef:794] [02:04:16.781] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:04:16.782+0000] [.info] [src/director/director.go:tracef:794] [02:04:16.781] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:04:17.009+0000] [.info] [src/director/director.go:tracef:794] [02:04:17.009] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:04:17.009+0000] [.info] [src/director/director.go:tracef:794] [02:04:17.009] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:04:17.009+0000] [.info] [src/director/director.go:tracef:794] [02:04:17.009] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:04:17.009+0000] [.info] [src/director/director.go:tracef:794] [02:04:17.009] because ['currentState.Up' = false] [2021-10-22T02:04:17.009+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:04:17.009] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:04:17.009+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:04:17.009] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:04:17.049+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:04:17.049] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:04:17.050+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:04:17.050] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 54694 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:04:17.120+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:04:17.120] clusterConfig unchanged [2021-10-22T02:04:17.737+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:17.737] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:17.825+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:04:17.825] ...Finished start [pid=25629] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54694,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:04:17.942+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:04:17.942] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 116.727626ms. [2021-10-22T02:04:17.942+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:04:17.942] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:54694 (local=false) [2021-10-22T02:04:17.942+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:04:17.942] Dropping existing documents from system.replset [2021-10-22T02:04:18.099+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:04:18.099] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:04:18.114+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:04:18.114] Inserting oplogTruncateAfterPoint document [2021-10-22T02:04:18.114+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:04:18.114] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:04:18.115+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:04:18.115] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:04:18.115+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:04:18.115] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:04:18.128+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:04:18.128] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:04:18.128+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:04:18.128] Setting the minvalid document to [{_id ObjectID("61721c22523370002becc754")} {t -1} {ts {0 1}}] [2021-10-22T02:04:18.140+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:04:18.140] Inserted [{_id ObjectID("61721c22523370002becc754")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:04:18.140+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:04:18.140] Inserting new replica set configuration document. [2021-10-22T02:04:18.140+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:04:18.140] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:04:18.154+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:04:18.154] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:04:18.169+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:04:18.169] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:04:18.169+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:04:18.169] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:04:18.169+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:04:18.169] Initiating stopEphemeralMongo [2021-10-22T02:04:18.169+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:04:18.169] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:54694 (local=false) [2021-10-22T02:04:18.169+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:04:18.169] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:54694 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:04:18.282+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:04:18.282] clusterConfig unchanged [2021-10-22T02:04:18.560+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:04:18.560] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.783475ms. [2021-10-22T02:04:18.560+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:04:18.560] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:54694 (local=false) [2021-10-22T02:04:18.560+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:04:18.560] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 22.926µs. [2021-10-22T02:04:18.560+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:04:18.560] Successfully finished running stopEphemeralMongo [2021-10-22T02:04:18.560+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:04:18.560] Successfully stopped existing ephemeral mongod. [2021-10-22T02:04:18.560+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:04:18.560] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:04:18.560+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:04:18.560] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:04:18.599+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:04:18.599] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:04:18.600+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:04:18.600] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57718 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:04:18.731+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:18.731] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:19.405+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:04:19.405] clusterConfig unchanged [2021-10-22T02:04:19.733+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:19.733] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:19.807+0000] [.error] [src/action/start.go:func1:93] [103] [02:04:19.807] Error starting mongod : [02:04:19.807] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57718 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:04:18.631Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 27081 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:04:18.664+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:04:18.664+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:04:18.664+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:04:18.666+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:04:18.668+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:04:18.685+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:04:18.716+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:04:18.716+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:04:18.717+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:04:18.763+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:04:18.763+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:04:19.284+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868259:284363][27081:0x7f6027591bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:04:19.388+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868259:388907][27081:0x7f6027591bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:04:19.495+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868259:495617][27081:0x7f6027591bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:04:19.615+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868259:615706][27081:0x7f6027591bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:04:19.693+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868259:693235][27081:0x7f6027591bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:04:19.755+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868259:755266][27081:0x7f6027591bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:04:19.755+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868259:755344][27081:0x7f6027591bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:04:19.807+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:04:19.807] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:57718 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:04:19.807] Error starting mongod : [02:04:19.807] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57718 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:04:18.631Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 27081 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:04:18.664+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:04:18.664+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:04:18.664+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:04:18.666+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:04:18.668+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:04:18.685+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:04:18.716+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:04:18.716+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:04:18.717+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:04:18.763+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:04:18.763+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:04:19.284+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868259:284363][27081:0x7f6027591bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:04:19.388+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:04:19.807+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:04:19.807] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:04:19.807] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:57718 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:04:19.807] Error starting mongod : [02:04:19.807] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57718 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:04:18.631Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 27081 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:04:18.664+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:04:18.664+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:04:18.664+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:04:18.666+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:04:18.668+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:04:18.685+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:04:18.716+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:04:18.716+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:04:18.717+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:04:18.763+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:04:18.763+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:04:19.284+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868259:284363][27081:0x7f6027591bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:04:19.807+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:04:19.807] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:04:19.807+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:04:19.807] Initiating stopEphemeralMongo [2021-10-22T02:04:19.807+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:04:19.807] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:04:19.808+0000] [.error] [src/director/director.go:executePlan:962] [02:04:19.808] Failed to apply action. Result = : [02:04:19.807] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:04:19.807] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:57718 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:04:19.807] Error starting mongod : [02:04:19.807] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57718 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:04:18.631Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 27081 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:04:18.664+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:04:18.664+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:04:18.664+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:04:18.666+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:04:18.668+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:04:18.685+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:04:18.716+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:04:18.716+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:04:18.717+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:04:18.763+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:04:18.763+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:04:19.284+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:04:19.808+0000] [.error] [src/director/director.go:planAndExecute:567] [02:04:19.808] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:04:19.808] Failed to apply action. Result = : [02:04:19.807] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:04:19.807] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:57718 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:04:19.807] Error starting mongod : [02:04:19.807] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57718 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:04:18.631Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 27081 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:04:18.664+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:04:18.664+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:04:18.664+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:04:18.666+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:04:18.668+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:04:18.685+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:04:18.716+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:04:18.716+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:04:18.717+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:04:18.763+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:04:18.763+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:04:19.808+0000] [.error] [src/director/director.go:mainLoop:399] [02:04:19.808] Failed to planAndExecute : [02:04:19.808] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:04:19.808] Failed to apply action. Result = : [02:04:19.807] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:04:19.807] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:57718 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:04:19.807] Error starting mongod : [02:04:19.807] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57718 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:04:18.631Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 27081 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:04:18.664+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:04:18.664+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:04:18.664+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57718,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:04:18.666+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:04:18.668+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:04:18.685+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:04:18.716+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:04:18.716+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:04:18.717+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:04:18.763+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:04:18.763+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:04:20.655+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:04:20.655] clusterConfig unchanged [2021-10-22T02:04:20.732+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:20.732] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:21.731+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:21.731] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:22.733+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:22.733] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:23.731+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:23.731] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:24.733+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:24.733] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:25.734+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:25.734] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:26.738+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:26.738] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:27.739+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:27.739] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:28.740+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:28.740] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:29.135+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:04:29.135] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 13.0360114s [2021-10-22T02:04:29.736+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:29.736] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:30.740+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:30.740] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:30.757+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:04:30.757] clusterConfig unchanged [2021-10-22T02:04:31.738+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:31.738] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:32.738+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:32.738] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:33.739+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:33.739] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:34.739+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:34.739] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:35.740+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:35.740] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:36.740+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:36.740] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:37.741+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:37.741] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:38.742+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:38.742] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:39.743+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:39.743] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:40.743+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:40.743] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:40.879+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:04:40.879] clusterConfig unchanged [2021-10-22T02:04:41.375+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:04:41.375] memory:rss=56254464 alloc=14136632 totalalloc=8925950264 mallocs=113810251 frees=113687070 heapinuse=18251776 heapobjects=123181 gcpausetotalns=1588170459 gcpauselastns=157129 gcnum=1339 utimens=185466797000 stimens=70956402000 minflt=474443 majflt=405 [2021-10-22T02:04:41.744+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:41.744] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:42.745+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:42.745] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:43.746+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:43.746] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:44.745+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:44.745] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:45.747+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:45.747] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:46.747+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:46.747] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:47.747+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:47.747] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:48.748+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:48.748] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:49.749+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:49.749] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:50.750+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:50.750] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:50.971+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:04:50.971] clusterConfig unchanged [2021-10-22T02:04:51.751+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:51.751] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:52.751+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:52.751] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:53.751+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:53.751] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:54.752+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:54.752] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:55.753+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:55.752] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:56.754+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:56.754] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:57.754+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:57.754] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:58.755+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:58.755] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:04:59.756+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:04:59.756] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:00.757+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:00.757] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:01.073+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:05:01.073] clusterConfig unchanged [2021-10-22T02:05:01.758+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:01.758] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:02.759+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:02.759] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:03.759+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:03.759] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:04.760+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:04.760] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:05.761+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:05.761] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:06.762+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:06.762] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:07.762+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:07.762] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:08.762+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:08.762] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:09.763+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:09.763] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:10.763+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:10.763] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:11.165+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:05:11.165] clusterConfig unchanged [2021-10-22T02:05:11.764+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:11.763] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:12.770+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:12.770] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:13.765+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:13.765] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:14.765+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:14.765] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:15.766+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:15.766] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:16.766+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:16.766] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:17.769+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:17.769] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:18.766+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:18.766] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:19.766+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:19.766] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:19.986+0000] [.info] [src/director/director.go:computePlan:280] [02:05:19.986] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:05:19.986+0000] [.info] [src/director/director.go:tracef:794] [02:05:19.986] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:05:19.986+0000] [.info] [src/director/director.go:tracef:794] [02:05:19.986] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:05:19.986+0000] [.info] [src/director/director.go:tracef:794] [02:05:19.986] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:05:19.986+0000] [.info] [src/director/director.go:tracef:794] [02:05:19.986] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:05:19.986+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:05:19.986] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:05:20.174+0000] [.info] [src/director/director.go:tracef:794] [02:05:20.174] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:05:20.174+0000] [.info] [src/director/director.go:tracef:794] [02:05:20.174] because [2021-10-22T02:05:20.174+0000] [.info] [src/director/director.go:tracef:794] [02:05:20.174] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:05:20.174+0000] [.info] [src/director/director.go:tracef:794] [02:05:20.174] because ['currentState.CanBounceStop' = true] [2021-10-22T02:05:20.342+0000] [.info] [src/director/director.go:tracef:794] [02:05:20.342] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:05:20.342+0000] [.info] [src/director/director.go:tracef:794] [02:05:20.342] because [2021-10-22T02:05:20.342+0000] [.info] [src/director/director.go:tracef:794] [02:05:20.342] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:05:20.342+0000] [.info] [src/director/director.go:tracef:794] [02:05:20.342] because ['currentState.Up' = false] [2021-10-22T02:05:20.342+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:05:20.342] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:05:20.342+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:05:20.342] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:05:20.421+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:05:20.421] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:05:20.429+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:05:20.429] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:05:20.429+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:05:20.429] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:05:20.430+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:05:20.430] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:05:20.431+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:05:20.431] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:05:20.431+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:05:20.431] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:05:20.431+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:05:20.431] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:05:20.433+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:05:20.433] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:05:20.657+0000] [.info] [src/director/director.go:tracef:794] [02:05:20.657] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:05:20.658+0000] [.info] [src/director/director.go:tracef:794] [02:05:20.658] because ['currentState.Up' = false] [2021-10-22T02:05:20.777+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:20.777] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:20.902+0000] [.info] [src/director/director.go:tracef:794] [02:05:20.902] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:05:20.902+0000] [.info] [src/director/director.go:tracef:794] [02:05:20.902] because ['currentState.Up' = false] [2021-10-22T02:05:20.902+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:05:20.902] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:05:21.134+0000] [.info] [src/director/director.go:tracef:794] [02:05:21.134] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:05:21.134+0000] [.info] [src/director/director.go:tracef:794] [02:05:21.134] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:05:21.341+0000] [.info] [src/director/director.go:tracef:794] [02:05:21.341] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:05:21.341+0000] [.info] [src/director/director.go:tracef:794] [02:05:21.341] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:05:21.341+0000] [.info] [src/director/director.go:tracef:794] [02:05:21.341] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:05:21.341+0000] [.info] [src/director/director.go:tracef:794] [02:05:21.341] because ['currentState.Up' = false] [2021-10-22T02:05:21.341+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:05:21.341] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:05:21.342+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:05:21.342] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:05:21.356+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:05:21.356] clusterConfig unchanged [2021-10-22T02:05:21.382+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:05:21.382] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:05:21.383+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:05:21.383] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 43594 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:05:21.789+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:21.789] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:22.160+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:05:22.160] ...Finished start [pid=46384] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":43594,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:05:22.272+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:05:22.272] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 111.932087ms. [2021-10-22T02:05:22.272+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:05:22.272] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:43594 (local=false) [2021-10-22T02:05:22.272+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:05:22.272] Dropping existing documents from system.replset [2021-10-22T02:05:22.406+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:05:22.406] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:05:22.412+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:05:22.412] Inserting oplogTruncateAfterPoint document [2021-10-22T02:05:22.412+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:05:22.412] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:05:22.413+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:05:22.413] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:05:22.413+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:05:22.413] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:05:22.426+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:05:22.426] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:05:22.426+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:05:22.426] Setting the minvalid document to [{_id ObjectID("61721c62523370002becc762")} {t -1} {ts {0 1}}] [2021-10-22T02:05:22.439+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:05:22.439] Inserted [{_id ObjectID("61721c62523370002becc762")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:05:22.439+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:05:22.439] Inserting new replica set configuration document. [2021-10-22T02:05:22.439+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:05:22.439] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:05:22.452+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:05:22.452] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:05:22.456+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:05:22.456] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:05:22.456+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:05:22.456] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:05:22.456+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:05:22.456] Initiating stopEphemeralMongo [2021-10-22T02:05:22.456+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:05:22.456] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:43594 (local=false) [2021-10-22T02:05:22.456+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:05:22.456] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:43594 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:05:22.470+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:05:22.470] clusterConfig unchanged [2021-10-22T02:05:22.788+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:22.788] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:22.830+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:05:22.830] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.336602ms. [2021-10-22T02:05:22.830+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:05:22.830] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:43594 (local=false) [2021-10-22T02:05:22.830+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:05:22.830] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.025µs. [2021-10-22T02:05:22.830+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:05:22.830] Successfully finished running stopEphemeralMongo [2021-10-22T02:05:22.830+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:05:22.830] Successfully stopped existing ephemeral mongod. [2021-10-22T02:05:22.830+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:05:22.830] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:05:22.830+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:05:22.830] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:05:22.864+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:05:22.864] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:05:22.865+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:05:22.865] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59709 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:05:23.572+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:05:23.571] clusterConfig unchanged [2021-10-22T02:05:23.792+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:23.792] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:24.280+0000] [.error] [src/action/start.go:func1:93] [103] [02:05:24.280] Error starting mongod : [02:05:24.279] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59709,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59709 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:05:22.897Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 47772 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:05:22.940+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:05:22.940+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:05:22.940+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59709,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:05:22.943+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:05:22.945+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:05:22.967+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:05:23.036+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:05:23.036+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:05:23.037+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:05:23.083+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:05:23.083+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:05:23.566+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868323:566077][47772:0x7fe604197bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:05:23.701+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868323:701466][47772:0x7fe604197bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:05:23.839+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868323:839042][47772:0x7fe604197bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/33920 to 2/256"}} {"t":{"$date":"2021-10-22T02:05:24.024+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868324:24513][47772:0x7fe604197bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:05:24.153+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868324:153876][47772:0x7fe604197bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:05:24.228+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868324:228105][47772:0x7fe604197bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:05:24.228+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868324:228181][47772:0x7fe604197bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:05:24.281+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:05:24.281] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59709 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:05:24.280] Error starting mongod : [02:05:24.279] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59709,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59709 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:05:22.897Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 47772 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:05:22.940+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:05:22.940+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:05:22.940+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59709,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:05:22.943+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:05:22.945+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:05:22.967+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:05:23.036+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:05:23.036+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:05:23.037+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:05:23.083+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:05:23.083+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:05:23.566+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868323:566077][47772:0x7fe604197bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:05:23.701+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:05:24.282+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:05:24.282] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:05:24.281] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59709 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:05:24.280] Error starting mongod : [02:05:24.279] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59709,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59709 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:05:22.897Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 47772 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:05:22.940+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:05:22.940+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:05:22.940+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59709,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:05:22.943+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:05:22.945+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:05:22.967+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:05:23.036+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:05:23.036+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:05:23.037+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:05:23.083+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:05:23.083+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:05:23.566+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868323:566077][47772:0x7fe604197bc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:05:24.283+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:05:24.283] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:05:24.283+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:05:24.283] Initiating stopEphemeralMongo [2021-10-22T02:05:24.283+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:05:24.283] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:05:24.292+0000] [.error] [src/director/director.go:executePlan:962] [02:05:24.292] Failed to apply action. Result = : [02:05:24.282] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:05:24.281] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59709 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:05:24.280] Error starting mongod : [02:05:24.279] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59709,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59709 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:05:22.897Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 47772 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:05:22.940+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:05:22.940+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:05:22.940+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59709,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:05:22.943+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:05:22.945+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:05:22.967+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:05:23.036+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:05:23.036+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:05:23.037+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:05:23.083+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:05:23.083+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:05:23.566+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:05:24.292+0000] [.error] [src/director/director.go:planAndExecute:567] [02:05:24.292] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:05:24.292] Failed to apply action. Result = : [02:05:24.282] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:05:24.281] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59709 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:05:24.280] Error starting mongod : [02:05:24.279] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59709,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59709 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:05:22.897Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 47772 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:05:22.940+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:05:22.940+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:05:22.940+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59709,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:05:22.943+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:05:22.945+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:05:22.967+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:05:23.036+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:05:23.036+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:05:23.037+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:05:23.083+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:05:23.083+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:05:24.292+0000] [.error] [src/director/director.go:mainLoop:399] [02:05:24.292] Failed to planAndExecute : [02:05:24.292] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:05:24.292] Failed to apply action. Result = : [02:05:24.282] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:05:24.281] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59709 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:05:24.280] Error starting mongod : [02:05:24.279] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59709,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59709 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:05:22.897Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 47772 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:05:22.940+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:05:22.940+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:05:22.940+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59709,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:05:22.943+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:05:22.945+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:05:22.967+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:05:23.036+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:05:23.036+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:05:23.037+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:05:23.083+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:05:23.083+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:05:24.793+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:24.793] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:24.842+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:05:24.842] clusterConfig unchanged [2021-10-22T02:05:25.793+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:25.793] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:26.795+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:26.795] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:27.801+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:27.801] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:28.802+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:28.802] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:29.803+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:29.803] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:30.804+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:30.804] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:31.803+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:31.803] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:32.805+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:32.805] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:33.292+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:05:33.291] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.858756198s [2021-10-22T02:05:33.806+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:33.806] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:34.802+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:34.802] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:34.974+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:05:34.974] clusterConfig unchanged [2021-10-22T02:05:35.803+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:35.803] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:36.804+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:36.804] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:37.804+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:37.804] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:38.805+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:38.805] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:39.806+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:39.806] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:40.806+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:40.806] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:41.376+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:05:41.376] memory:rss=56778752 alloc=14504720 totalalloc=9017346720 mallocs=114982638 frees=114857259 heapinuse=18497536 heapobjects=125379 gcpausetotalns=1590839678 gcpauselastns=153297 gcnum=1351 utimens=187875281000 stimens=71962808000 minflt=479369 majflt=405 [2021-10-22T02:05:41.806+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:41.806] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:42.807+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:42.807] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:43.807+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:43.807] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:44.372+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [02:05:44.372] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:59709 (local=false) because it is not up [2021-10-22T02:05:44.808+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:44.808] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:45.106+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:05:45.106] clusterConfig unchanged [2021-10-22T02:05:45.809+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:45.809] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:46.809+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:46.809] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:47.810+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:47.810] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:48.810+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:48.810] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:49.824+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:49.824] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:50.812+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:50.812] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:51.813+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:51.813] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:52.814+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:52.814] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:53.814+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:53.814] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:54.815+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:54.815] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:55.217+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:05:55.217] clusterConfig unchanged [2021-10-22T02:05:55.815+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:55.815] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:56.815+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:56.815] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:57.815+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:57.815] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:58.817+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:58.817] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:05:59.817+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:05:59.817] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:00.817+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:00.817] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:01.818+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:01.818] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:02.819+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:02.819] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:03.826+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:03.825] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:04.823+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:04.823] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:05.310+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:06:05.310] clusterConfig unchanged [2021-10-22T02:06:05.820+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:05.820] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:06.821+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:06.821] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:07.827+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:07.827] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:08.822+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:08.822] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:09.823+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:09.823] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:10.824+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:10.824] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:11.824+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:11.824] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:12.825+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:12.825] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:13.826+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:13.826] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:14.826+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:14.826] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:15.422+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:06:15.422] clusterConfig unchanged [2021-10-22T02:06:15.833+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:15.833] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:16.827+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:16.827] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:17.828+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:17.828] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:18.564+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:06:18.564] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:06:18.566+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:18.566] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:06:18.566+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [02:06:18.566] Could not get version : [02:06:18.566] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:06:18.566+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [02:06:18.566] Error getting version for log lines. Skipping : [02:06:18.566] Could not get version : [02:06:18.566] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:06:18.829+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:18.829] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:19.830+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:19.830] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:20.830+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:20.830] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:21.831+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:21.831] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:22.832+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:22.832] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:23.832+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:23.832] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:24.468+0000] [.info] [src/director/director.go:computePlan:280] [02:06:24.468] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:06:24.468+0000] [.info] [src/director/director.go:tracef:794] [02:06:24.468] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:06:24.468+0000] [.info] [src/director/director.go:tracef:794] [02:06:24.468] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:06:24.469+0000] [.info] [src/director/director.go:tracef:794] [02:06:24.469] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:06:24.469+0000] [.info] [src/director/director.go:tracef:794] [02:06:24.469] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:06:24.469+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:06:24.469] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:06:24.623+0000] [.info] [src/director/director.go:tracef:794] [02:06:24.623] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:06:24.623+0000] [.info] [src/director/director.go:tracef:794] [02:06:24.623] because [2021-10-22T02:06:24.624+0000] [.info] [src/director/director.go:tracef:794] [02:06:24.624] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:06:24.624+0000] [.info] [src/director/director.go:tracef:794] [02:06:24.624] because ['currentState.CanBounceStop' = true] [2021-10-22T02:06:24.812+0000] [.info] [src/director/director.go:tracef:794] [02:06:24.812] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:06:24.812+0000] [.info] [src/director/director.go:tracef:794] [02:06:24.812] because [2021-10-22T02:06:24.812+0000] [.info] [src/director/director.go:tracef:794] [02:06:24.812] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:06:24.812+0000] [.info] [src/director/director.go:tracef:794] [02:06:24.812] because ['currentState.Up' = false] [2021-10-22T02:06:24.812+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:06:24.812] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:06:24.812+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:06:24.812] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:06:24.833+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:24.833] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:24.894+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:06:24.894] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:06:24.902+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:06:24.902] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:06:24.903+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:06:24.903] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:06:24.904+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:06:24.904] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:06:24.904+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:06:24.904] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:06:24.904+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:06:24.904] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:06:24.905+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:06:24.905] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:06:25.137+0000] [.info] [src/director/director.go:tracef:794] [02:06:25.137] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:06:25.137+0000] [.info] [src/director/director.go:tracef:794] [02:06:25.137] because ['currentState.Up' = false] [2021-10-22T02:06:25.338+0000] [.info] [src/director/director.go:tracef:794] [02:06:25.338] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:06:25.338+0000] [.info] [src/director/director.go:tracef:794] [02:06:25.338] because ['currentState.Up' = false] [2021-10-22T02:06:25.339+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:06:25.338] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:06:25.592+0000] [.info] [src/director/director.go:tracef:794] [02:06:25.592] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:06:25.592+0000] [.info] [src/director/director.go:tracef:794] [02:06:25.592] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:06:25.595+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:06:25.595] clusterConfig unchanged [2021-10-22T02:06:25.790+0000] [.info] [src/director/director.go:tracef:794] [02:06:25.790] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:06:25.790+0000] [.info] [src/director/director.go:tracef:794] [02:06:25.790] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:06:25.791+0000] [.info] [src/director/director.go:tracef:794] [02:06:25.791] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:06:25.791+0000] [.info] [src/director/director.go:tracef:794] [02:06:25.791] because ['currentState.Up' = false] [2021-10-22T02:06:25.791+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:06:25.791] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:06:25.791+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:06:25.791] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:06:25.836+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:06:25.836] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:06:25.837+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:06:25.837] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40803 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:06:25.839+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:25.839] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:26.590+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:06:26.590] ...Finished start [pid=67151] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40803,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:06:26.722+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:06:26.722] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 132.364476ms. [2021-10-22T02:06:26.722+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:06:26.722] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40803 (local=false) [2021-10-22T02:06:26.722+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:06:26.722] Dropping existing documents from system.replset [2021-10-22T02:06:26.725+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:06:26.725] clusterConfig unchanged [2021-10-22T02:06:26.840+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:26.840] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:26.876+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:06:26.876] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:06:26.880+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:06:26.880] Inserting oplogTruncateAfterPoint document [2021-10-22T02:06:26.880+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:06:26.880] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:06:26.881+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:06:26.881] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:06:26.881+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:06:26.881] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:06:26.894+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:06:26.894] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:06:26.894+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:06:26.894] Setting the minvalid document to [{_id ObjectID("61721ca2523370002becc770")} {t -1} {ts {0 1}}] [2021-10-22T02:06:26.906+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:06:26.906] Inserted [{_id ObjectID("61721ca2523370002becc770")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:06:26.906+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:06:26.906] Inserting new replica set configuration document. [2021-10-22T02:06:26.906+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:06:26.906] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:06:26.928+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:06:26.928] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:06:26.932+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:06:26.932] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:06:26.932+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:06:26.932] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:06:26.932+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:06:26.932] Initiating stopEphemeralMongo [2021-10-22T02:06:26.932+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:06:26.932] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40803 (local=false) [2021-10-22T02:06:26.932+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:06:26.932] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40803 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:06:27.304+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:06:27.304] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.11968ms. [2021-10-22T02:06:27.304+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:06:27.304] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40803 (local=false) [2021-10-22T02:06:27.304+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:06:27.304] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.405µs. [2021-10-22T02:06:27.304+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:06:27.304] Successfully finished running stopEphemeralMongo [2021-10-22T02:06:27.304+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:06:27.304] Successfully stopped existing ephemeral mongod. [2021-10-22T02:06:27.304+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:06:27.304] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:06:27.304+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:06:27.304] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:06:27.339+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:06:27.339] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:06:27.340+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:06:27.340] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40328 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:06:27.845+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:27.845] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:27.856+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:06:27.856] clusterConfig unchanged [2021-10-22T02:06:28.567+0000] [.error] [src/action/start.go:func1:93] [103] [02:06:28.567] Error starting mongod : [02:06:28.567] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40328,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40328 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 68685 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:06:27.392+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:06:27.392+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:06:27.392+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40328,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:06:27.394+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:06:27.395+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:06:27.397+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:06:27.417+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:06:27.417+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:06:27.417+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:06:27.463+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:06:27.463+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:06:28.062+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868388:62939][68685:0x7f17b1597bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:06:28.165+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868388:165812][68685:0x7f17b1597bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:06:28.268+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868388:268744][68685:0x7f17b1597bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:06:28.383+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868388:383243][68685:0x7f17b1597bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:06:28.457+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868388:457349][68685:0x7f17b1597bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:06:28.517+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868388:517470][68685:0x7f17b1597bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:06:28.517+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868388:517544][68685:0x7f17b1597bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:06:28.519+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:06:28.567+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:06:28.567] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:40328 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:06:28.567] Error starting mongod : [02:06:28.567] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40328,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40328 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 68685 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:06:27.392+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:06:27.392+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:06:27.392+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40328,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:06:27.394+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:06:27.395+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:06:27.397+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:06:27.417+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:06:27.417+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:06:27.417+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:06:27.463+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:06:27.463+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:06:28.062+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868388:62939][68685:0x7f17b1597bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:06:28.165+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868388:165812][68685:0x7f17b1597bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:06:28.568+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:06:28.568] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:06:28.567] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:40328 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:06:28.567] Error starting mongod : [02:06:28.567] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40328,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40328 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 68685 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:06:27.392+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:06:27.392+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:06:27.392+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40328,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:06:27.394+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:06:27.395+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:06:27.397+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:06:27.417+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:06:27.417+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:06:27.417+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:06:27.463+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:06:27.463+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:06:28.062+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868388:62939][68685:0x7f17b1597bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:06:28.165+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:06:28.568+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:06:28.568] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:06:28.568+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:06:28.568] Initiating stopEphemeralMongo [2021-10-22T02:06:28.568+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:06:28.568] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:06:28.568+0000] [.error] [src/director/director.go:executePlan:962] [02:06:28.568] Failed to apply action. Result = : [02:06:28.568] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:06:28.567] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:40328 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:06:28.567] Error starting mongod : [02:06:28.567] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40328,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40328 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 68685 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:06:27.392+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:06:27.392+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:06:27.392+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40328,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:06:27.394+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:06:27.395+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:06:27.397+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:06:27.417+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:06:27.417+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:06:27.417+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:06:27.463+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:06:27.463+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:06:28.062+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868388:62939][68685:0x7f17b1597bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:06:28.569+0000] [.error] [src/director/director.go:planAndExecute:567] [02:06:28.568] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:06:28.568] Failed to apply action. Result = : [02:06:28.568] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:06:28.567] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:40328 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:06:28.567] Error starting mongod : [02:06:28.567] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40328,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40328 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 68685 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:06:27.392+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:06:27.392+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:06:27.392+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40328,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:06:27.394+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:06:27.395+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:06:27.397+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:06:27.417+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:06:27.417+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:06:27.417+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:06:27.463+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:06:27.463+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:06:28.062+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:06:28.569+0000] [.error] [src/director/director.go:mainLoop:399] [02:06:28.569] Failed to planAndExecute : [02:06:28.568] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:06:28.568] Failed to apply action. Result = : [02:06:28.568] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:06:28.567] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:40328 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:06:28.567] Error starting mongod : [02:06:28.567] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40328,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40328 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 68685 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:06:27.392+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:06:27.392+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:06:27.392+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40328,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:06:27.394+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:06:27.395+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:06:27.397+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:06:27.417+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:06:27.417+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:06:27.417+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:06:27.463+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:06:27.463+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:06:28.062+00:00"},"s":"I", "c":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:06:28.846+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:28.846] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:29.139+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:06:29.139] clusterConfig unchanged [2021-10-22T02:06:29.847+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:29.847] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:30.847+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:30.847] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:31.848+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:31.848] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:32.848+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:32.848] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:33.849+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:33.849] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:34.849+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:34.849] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:35.851+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:35.851] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:36.854+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:36.854] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:37.650+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:06:37.650] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.744099115s [2021-10-22T02:06:37.857+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:37.857] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:38.859+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:38.859] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:39.239+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:06:39.239] clusterConfig unchanged [2021-10-22T02:06:39.853+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:39.853] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:40.854+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:40.854] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:41.376+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:06:41.376] memory:rss=57032704 alloc=9559664 totalalloc=9110501688 mallocs=116163026 frees=116091529 heapinuse=17072128 heapobjects=71497 gcpausetotalns=1592118218 gcpauselastns=93623 gcnum=1364 utimens=190122646000 stimens=72991941000 minflt=484876 majflt=405 [2021-10-22T02:06:41.856+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:41.856] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:42.856+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:42.856] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:43.857+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:43.857] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:44.858+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:44.858] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:45.857+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:45.857] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:46.857+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:46.857] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:47.858+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:47.858] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:48.868+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:48.868] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:49.362+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:06:49.362] clusterConfig unchanged [2021-10-22T02:06:49.860+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:49.860] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:50.861+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:50.861] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:51.861+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:51.861] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:52.861+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:52.861] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:53.862+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:53.862] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:54.862+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:54.862] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:55.864+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:55.864] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:56.864+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:56.864] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:57.864+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:57.864] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:58.865+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:58.865] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:06:59.483+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:06:59.483] clusterConfig unchanged [2021-10-22T02:06:59.865+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:06:59.865] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:00.874+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:00.874] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:01.872+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:01.872] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:02.867+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:02.867] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:03.867+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:03.867] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:04.868+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:04.868] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:05.868+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:05.868] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:06.868+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:06.868] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:07.869+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:07.869] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:08.869+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:08.869] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:09.615+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:07:09.615] clusterConfig unchanged [2021-10-22T02:07:09.885+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:09.885] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:10.873+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:10.873] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:11.871+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:11.871] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:12.872+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:12.872] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:13.873+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:13.873] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:14.874+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:14.874] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:15.875+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:15.875] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:16.875+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:16.875] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:17.876+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:17.876] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:18.876+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:18.876] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:19.845+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:07:19.845] clusterConfig unchanged [2021-10-22T02:07:19.876+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:19.876] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:20.877+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:20.877] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:21.879+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:21.879] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:22.877+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:22.877] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:23.879+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:23.879] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:24.880+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:24.880] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:25.881+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:25.881] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:26.884+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:26.884] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:27.569+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:07:27.569] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:07:27.570+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:07:27.570] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:07:27.882+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:27.882] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:28.742+0000] [.info] [src/director/director.go:computePlan:280] [02:07:28.742] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:07:28.742+0000] [.info] [src/director/director.go:tracef:794] [02:07:28.742] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:07:28.742+0000] [.info] [src/director/director.go:tracef:794] [02:07:28.742] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:07:28.742+0000] [.info] [src/director/director.go:tracef:794] [02:07:28.742] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:07:28.742+0000] [.info] [src/director/director.go:tracef:794] [02:07:28.742] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:07:28.742+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:07:28.742] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:07:28.885+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:28.885] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:28.907+0000] [.info] [src/director/director.go:tracef:794] [02:07:28.907] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:07:28.907+0000] [.info] [src/director/director.go:tracef:794] [02:07:28.907] because [2021-10-22T02:07:28.907+0000] [.info] [src/director/director.go:tracef:794] [02:07:28.907] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:07:28.907+0000] [.info] [src/director/director.go:tracef:794] [02:07:28.907] because ['currentState.CanBounceStop' = true] [2021-10-22T02:07:29.066+0000] [.info] [src/director/director.go:tracef:794] [02:07:29.066] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:07:29.066+0000] [.info] [src/director/director.go:tracef:794] [02:07:29.066] because [2021-10-22T02:07:29.066+0000] [.info] [src/director/director.go:tracef:794] [02:07:29.066] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:07:29.066+0000] [.info] [src/director/director.go:tracef:794] [02:07:29.066] because ['currentState.Up' = false] [2021-10-22T02:07:29.066+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:07:29.066] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:07:29.066+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:07:29.066] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:07:29.170+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:07:29.170] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:07:29.178+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:07:29.178] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:07:29.178+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:07:29.178] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:07:29.179+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:07:29.179] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:07:29.180+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:07:29.180] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:07:29.180+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:07:29.180] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:07:29.180+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:07:29.180] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:07:29.181+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:07:29.181] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:07:29.415+0000] [.info] [src/director/director.go:tracef:794] [02:07:29.414] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:07:29.415+0000] [.info] [src/director/director.go:tracef:794] [02:07:29.415] because ['currentState.Up' = false] [2021-10-22T02:07:29.662+0000] [.info] [src/director/director.go:tracef:794] [02:07:29.662] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:07:29.662+0000] [.info] [src/director/director.go:tracef:794] [02:07:29.662] because ['currentState.Up' = false] [2021-10-22T02:07:29.662+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:07:29.662] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:07:29.881+0000] [.info] [src/director/director.go:tracef:794] [02:07:29.881] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:07:29.882+0000] [.info] [src/director/director.go:tracef:794] [02:07:29.882] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:07:29.887+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:29.887] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:30.054+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:07:30.054] clusterConfig unchanged [2021-10-22T02:07:30.112+0000] [.info] [src/director/director.go:tracef:794] [02:07:30.112] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:07:30.112+0000] [.info] [src/director/director.go:tracef:794] [02:07:30.112] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:07:30.113+0000] [.info] [src/director/director.go:tracef:794] [02:07:30.113] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:07:30.113+0000] [.info] [src/director/director.go:tracef:794] [02:07:30.113] because ['currentState.Up' = false] [2021-10-22T02:07:30.113+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:07:30.113] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:07:30.113+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:07:30.113] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:07:30.149+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:07:30.149] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:07:30.150+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:07:30.150] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36331 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:07:30.886+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:30.886] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:30.936+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:07:30.936] ...Finished start [pid=13938] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36331,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:07:31.072+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:07:31.072] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 135.604028ms. [2021-10-22T02:07:31.072+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:07:31.072] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36331 (local=false) [2021-10-22T02:07:31.072+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:07:31.072] Dropping existing documents from system.replset [2021-10-22T02:07:31.180+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:07:31.180] clusterConfig unchanged [2021-10-22T02:07:31.240+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:07:31.240] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:07:31.244+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:07:31.244] Inserting oplogTruncateAfterPoint document [2021-10-22T02:07:31.244+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:07:31.244] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:07:31.245+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:07:31.245] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:07:31.245+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:07:31.245] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:07:31.259+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:07:31.259] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:07:31.259+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:07:31.259] Setting the minvalid document to [{_id ObjectID("61721ce3523370002becc77e")} {t -1} {ts {0 1}}] [2021-10-22T02:07:31.272+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:07:31.272] Inserted [{_id ObjectID("61721ce3523370002becc77e")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:07:31.272+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:07:31.272] Inserting new replica set configuration document. [2021-10-22T02:07:31.272+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:07:31.272] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:07:31.284+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:07:31.284] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:07:31.289+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:07:31.289] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:07:31.289+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:07:31.289] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:07:31.289+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:07:31.289] Initiating stopEphemeralMongo [2021-10-22T02:07:31.289+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:07:31.289] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36331 (local=false) [2021-10-22T02:07:31.289+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:07:31.289] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36331 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:07:31.787+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:07:31.787] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.155904ms. [2021-10-22T02:07:31.787+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:07:31.787] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36331 (local=false) [2021-10-22T02:07:31.788+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:07:31.787] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.666µs. [2021-10-22T02:07:31.788+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:07:31.788] Successfully finished running stopEphemeralMongo [2021-10-22T02:07:31.788+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:07:31.788] Successfully stopped existing ephemeral mongod. [2021-10-22T02:07:31.788+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:07:31.788] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:07:31.788+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:07:31.788] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:07:31.820+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:07:31.820] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:07:31.821+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:07:31.821] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49769 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:07:31.898+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:31.898] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:32.311+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:07:32.311] clusterConfig unchanged [2021-10-22T02:07:32.900+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:32.900] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:33.097+0000] [.error] [src/action/start.go:func1:93] [103] [02:07:33.097] Error starting mongod : [02:07:33.097] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49769,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49769 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:07:31.855Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 15409 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:07:31.872+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:07:31.872+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:07:31.872+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49769,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:07:31.874+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:07:31.875+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:07:31.878+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:07:31.916+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:07:31.916+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:07:31.916+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:07:31.970+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:07:31.970+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:07:32.537+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868452:537361][15409:0x7f4abd583bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:07:32.680+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868452:680488][15409:0x7f4abd583bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:07:32.792+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868452:792474][15409:0x7f4abd583bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:07:32.918+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868452:918115][15409:0x7f4abd583bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:07:32.991+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868452:991890][15409:0x7f4abd583bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:07:33.049+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868453:49428][15409:0x7f4abd583bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:07:33.049+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868453:49508][15409:0x7f4abd583bc0], txn-recover: [WT_VERB_RECOVERY | WT_...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:07:33.097+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:07:33.097] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49769 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:07:33.097] Error starting mongod : [02:07:33.097] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49769,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49769 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:07:31.855Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 15409 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:07:31.872+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:07:31.872+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:07:31.872+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49769,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:07:31.874+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:07:31.875+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:07:31.878+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:07:31.916+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:07:31.916+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:07:31.916+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:07:31.970+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:07:31.970+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:07:32.537+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868452:537361][15409:0x7f4abd583bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:07:32.680+00:00"},"s":"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:07:33.098+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:07:33.098] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:07:33.097] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49769 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:07:33.097] Error starting mongod : [02:07:33.097] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49769,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49769 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:07:31.855Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 15409 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:07:31.872+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:07:31.872+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:07:31.872+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49769,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:07:31.874+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:07:31.875+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:07:31.878+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:07:31.916+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:07:31.916+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:07:31.916+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:07:31.970+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:07:31.970+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:07:32.537+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868452:537361][15409:0x7f4abd583bc0], ... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:07:33.098+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:07:33.098] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:07:33.098+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:07:33.098] Initiating stopEphemeralMongo [2021-10-22T02:07:33.098+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:07:33.098] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:07:33.098+0000] [.error] [src/director/director.go:executePlan:962] [02:07:33.098] Failed to apply action. Result = : [02:07:33.098] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:07:33.097] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49769 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:07:33.097] Error starting mongod : [02:07:33.097] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49769,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49769 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:07:31.855Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 15409 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:07:31.872+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:07:31.872+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:07:31.872+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49769,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:07:31.874+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:07:31.875+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:07:31.878+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:07:31.916+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:07:31.916+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:07:31.916+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:07:31.970+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:07:31.970+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:07:32.537+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","m... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:07:33.098+0000] [.error] [src/director/director.go:planAndExecute:567] [02:07:33.098] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:07:33.098] Failed to apply action. Result = : [02:07:33.098] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:07:33.097] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49769 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:07:33.097] Error starting mongod : [02:07:33.097] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49769,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49769 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:07:31.855Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 15409 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:07:31.872+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:07:31.872+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:07:31.872+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49769,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:07:31.874+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:07:31.875+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:07:31.878+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:07:31.916+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:07:31.916+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:07:31.916+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:07:31.970+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:07:31.970+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:07:33.099+0000] [.error] [src/director/director.go:mainLoop:399] [02:07:33.098] Failed to planAndExecute : [02:07:33.098] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:07:33.098] Failed to apply action. Result = : [02:07:33.098] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:07:33.097] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49769 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:07:33.097] Error starting mongod : [02:07:33.097] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49769,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49769 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:07:31.855Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 15409 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:07:31.872+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:07:31.872+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:07:31.872+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49769,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:07:31.874+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:07:31.875+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:07:31.878+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:07:31.916+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:07:31.916+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:07:31.916+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:07:31.970+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:07:31.970+00:00"},"s":"I", "c":"NETWORK", "id":577... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:07:33.561+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:07:33.561] clusterConfig unchanged [2021-10-22T02:07:33.898+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:33.898] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:34.898+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:34.898] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:35.913+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:35.913] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:36.899+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:36.899] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:37.901+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [02:07:37.901] lsblk cache invalidated, running lsblk command [2021-10-22T02:07:37.903+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:37.903] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:38.898+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:38.898] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:39.902+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:39.902] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:40.900+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:40.900] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:41.377+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:07:41.377] memory:rss=57393152 alloc=11084416 totalalloc=9201411864 mallocs=117332728 frees=117249813 heapinuse=17481728 heapobjects=82915 gcpausetotalns=1594124867 gcpauselastns=148271 gcnum=1376 utimens=192467008000 stimens=74023432000 minflt=489556 majflt=405 [2021-10-22T02:07:41.904+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:41.904] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:42.189+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:07:42.189] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 13.007696941s [2021-10-22T02:07:42.900+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:42.900] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:43.702+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:07:43.702] clusterConfig unchanged [2021-10-22T02:07:43.901+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:43.901] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:44.902+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:44.902] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:45.902+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:45.902] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:46.902+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:46.902] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:47.904+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:47.904] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:48.910+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:48.910] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:49.904+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:49.904] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:50.905+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:50.905] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:51.905+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:51.905] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:52.906+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:52.906] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:53.795+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:07:53.795] clusterConfig unchanged [2021-10-22T02:07:53.906+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:53.906] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:54.907+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:54.907] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:55.907+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:55.907] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:56.907+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:56.907] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:57.907+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:57.907] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:58.908+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:58.908] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:07:59.909+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:07:59.909] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:00.910+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:00.910] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:01.911+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:01.911] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:02.917+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:02.917] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:03.912+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:03.912] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:03.927+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:08:03.927] clusterConfig unchanged [2021-10-22T02:08:04.913+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:04.913] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:05.913+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:05.913] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:06.914+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:06.914] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:07.915+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:07.915] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:08.915+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:08.915] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:09.915+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:09.915] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:10.917+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:10.917] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:11.917+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:11.917] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:12.917+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:12.917] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:13.917+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:13.917] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:14.029+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:08:14.029] clusterConfig unchanged [2021-10-22T02:08:14.918+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:14.918] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:15.918+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:15.918] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:16.919+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:16.919] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:17.921+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:17.921] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:18.921+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:18.921] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:19.921+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:19.921] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:20.921+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:20.921] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:21.924+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:21.924] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:22.921+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:22.921] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:23.927+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:23.927] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:24.170+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:08:24.170] clusterConfig unchanged [2021-10-22T02:08:24.938+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:24.938] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:25.923+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:25.923] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:26.924+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:26.924] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:27.925+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:27.925] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:28.925+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:28.925] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:29.925+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:29.925] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:30.925+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:30.925] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:31.926+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:31.926] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:32.926+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:32.926] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:33.275+0000] [.info] [src/director/director.go:computePlan:280] [02:08:33.275] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:08:33.276+0000] [.info] [src/director/director.go:tracef:794] [02:08:33.276] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:08:33.276+0000] [.info] [src/director/director.go:tracef:794] [02:08:33.276] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:08:33.276+0000] [.info] [src/director/director.go:tracef:794] [02:08:33.276] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:08:33.276+0000] [.info] [src/director/director.go:tracef:794] [02:08:33.276] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:08:33.276+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:08:33.276] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:08:33.434+0000] [.info] [src/director/director.go:tracef:794] [02:08:33.434] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:08:33.434+0000] [.info] [src/director/director.go:tracef:794] [02:08:33.434] because [2021-10-22T02:08:33.434+0000] [.info] [src/director/director.go:tracef:794] [02:08:33.434] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:08:33.434+0000] [.info] [src/director/director.go:tracef:794] [02:08:33.434] because ['currentState.CanBounceStop' = true] [2021-10-22T02:08:33.596+0000] [.info] [src/director/director.go:tracef:794] [02:08:33.596] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:08:33.596+0000] [.info] [src/director/director.go:tracef:794] [02:08:33.596] because [2021-10-22T02:08:33.596+0000] [.info] [src/director/director.go:tracef:794] [02:08:33.596] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:08:33.596+0000] [.info] [src/director/director.go:tracef:794] [02:08:33.596] because ['currentState.Up' = false] [2021-10-22T02:08:33.596+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:08:33.596] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:08:33.596+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:08:33.596] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:08:33.682+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:08:33.682] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:08:33.691+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:08:33.691] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:08:33.691+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:08:33.691] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:08:33.692+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:08:33.692] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:08:33.693+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:08:33.693] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:08:33.693+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:08:33.693] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:08:33.693+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:08:33.693] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:08:33.694+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:08:33.694] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:08:33.897+0000] [.info] [src/director/director.go:tracef:794] [02:08:33.896] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:08:33.897+0000] [.info] [src/director/director.go:tracef:794] [02:08:33.897] because ['currentState.Up' = false] [2021-10-22T02:08:33.938+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:33.938] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:34.146+0000] [.info] [src/director/director.go:tracef:794] [02:08:34.146] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:08:34.146+0000] [.info] [src/director/director.go:tracef:794] [02:08:34.146] because ['currentState.Up' = false] [2021-10-22T02:08:34.146+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:08:34.146] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:08:34.349+0000] [.info] [src/director/director.go:tracef:794] [02:08:34.348] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:08:34.349+0000] [.info] [src/director/director.go:tracef:794] [02:08:34.349] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:08:34.371+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:08:34.371] clusterConfig unchanged [2021-10-22T02:08:34.559+0000] [.info] [src/director/director.go:tracef:794] [02:08:34.559] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:08:34.559+0000] [.info] [src/director/director.go:tracef:794] [02:08:34.559] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:08:34.559+0000] [.info] [src/director/director.go:tracef:794] [02:08:34.559] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:08:34.559+0000] [.info] [src/director/director.go:tracef:794] [02:08:34.559] because ['currentState.Up' = false] [2021-10-22T02:08:34.559+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:08:34.559] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:08:34.560+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:08:34.559] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:08:34.595+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:08:34.595] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:08:34.596+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:08:34.596] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56739 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:08:34.946+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:34.946] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:35.348+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:08:35.348] ...Finished start [pid=34587] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56739,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:08:35.493+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:08:35.493] clusterConfig unchanged [2021-10-22T02:08:35.493+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:08:35.493] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 145.429763ms. [2021-10-22T02:08:35.494+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:08:35.494] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:56739 (local=false) [2021-10-22T02:08:35.494+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:08:35.494] Dropping existing documents from system.replset [2021-10-22T02:08:35.618+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:08:35.618] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:08:35.633+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:08:35.633] Inserting oplogTruncateAfterPoint document [2021-10-22T02:08:35.633+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:08:35.633] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:08:35.635+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:08:35.635] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:08:35.635+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:08:35.635] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:08:35.648+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:08:35.648] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:08:35.648+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:08:35.648] Setting the minvalid document to [{_id ObjectID("61721d23523370002becc78c")} {t -1} {ts {0 1}}] [2021-10-22T02:08:35.660+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:08:35.660] Inserted [{_id ObjectID("61721d23523370002becc78c")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:08:35.660+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:08:35.660] Inserting new replica set configuration document. [2021-10-22T02:08:35.660+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:08:35.660] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:08:35.672+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:08:35.672] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:08:35.676+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:08:35.676] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:08:35.676+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:08:35.676] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:08:35.676+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:08:35.676] Initiating stopEphemeralMongo [2021-10-22T02:08:35.676+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:08:35.676] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:56739 (local=false) [2021-10-22T02:08:35.676+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:08:35.676] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:56739 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:08:35.946+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:35.946] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:36.074+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:08:36.074] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.147051ms. [2021-10-22T02:08:36.074+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:08:36.074] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:56739 (local=false) [2021-10-22T02:08:36.074+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:08:36.074] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 38.388µs. [2021-10-22T02:08:36.074+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:08:36.074] Successfully finished running stopEphemeralMongo [2021-10-22T02:08:36.074+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:08:36.074] Successfully stopped existing ephemeral mongod. [2021-10-22T02:08:36.074+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:08:36.074] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:08:36.074+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:08:36.074] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:08:36.109+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:08:36.109] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:08:36.110+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:08:36.110] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34099 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:08:36.616+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:08:36.616] clusterConfig unchanged [2021-10-22T02:08:36.947+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:36.947] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:37.366+0000] [.error] [src/action/start.go:func1:93] [103] [02:08:37.366] Error starting mongod : [02:08:37.366] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34099,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34099 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:08:36.140Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 36132 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:08:36.159+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:08:36.159+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:08:36.159+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34099,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:08:36.160+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:08:36.163+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:08:36.164+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:08:36.183+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:08:36.183+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:08:36.183+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:08:36.228+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:08:36.228+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:08:36.774+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868516:774632][36132:0x7f935615dbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:08:36.903+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868516:903030][36132:0x7f935615dbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:08:37.012+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868517:12824][36132:0x7f935615dbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:08:37.140+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868517:140459][36132:0x7f935615dbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:08:37.215+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868517:215131][36132:0x7f935615dbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:08:37.315+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868517:315407][36132:0x7f935615dbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:08:37.315+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868517:315480][36132:0x7f935615dbc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:08:37.366+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:08:37.366] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34099 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:08:37.366] Error starting mongod : [02:08:37.366] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34099,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34099 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:08:36.140Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 36132 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:08:36.159+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:08:36.159+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:08:36.159+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34099,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:08:36.160+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:08:36.163+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:08:36.164+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:08:36.183+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:08:36.183+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:08:36.183+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:08:36.228+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:08:36.228+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:08:36.774+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868516:774632][36132:0x7f935615dbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:08:36.903+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:08:37.366+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:08:37.366] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:08:37.366] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34099 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:08:37.366] Error starting mongod : [02:08:37.366] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34099,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34099 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:08:36.140Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 36132 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:08:36.159+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:08:36.159+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:08:36.159+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34099,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:08:36.160+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:08:36.163+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:08:36.164+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:08:36.183+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:08:36.183+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:08:36.183+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:08:36.228+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:08:36.228+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:08:36.774+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868516:774632][36132:0x7f935615dbc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:08:37.367+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:08:37.367] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:08:37.367+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:08:37.367] Initiating stopEphemeralMongo [2021-10-22T02:08:37.367+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:08:37.367] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:08:37.367+0000] [.error] [src/director/director.go:executePlan:962] [02:08:37.367] Failed to apply action. Result = : [02:08:37.366] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:08:37.366] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34099 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:08:37.366] Error starting mongod : [02:08:37.366] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34099,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34099 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:08:36.140Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 36132 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:08:36.159+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:08:36.159+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:08:36.159+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34099,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:08:36.160+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:08:36.163+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:08:36.164+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:08:36.183+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:08:36.183+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:08:36.183+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:08:36.228+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:08:36.228+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:08:36.774+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:08:37.367+0000] [.error] [src/director/director.go:planAndExecute:567] [02:08:37.367] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:08:37.367] Failed to apply action. Result = : [02:08:37.366] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:08:37.366] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34099 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:08:37.366] Error starting mongod : [02:08:37.366] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34099,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34099 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:08:36.140Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 36132 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:08:36.159+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:08:36.159+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:08:36.159+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34099,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:08:36.160+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:08:36.163+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:08:36.164+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:08:36.183+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:08:36.183+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:08:36.183+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:08:36.228+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:08:36.228+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:08:37.367+0000] [.error] [src/director/director.go:mainLoop:399] [02:08:37.367] Failed to planAndExecute : [02:08:37.367] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:08:37.367] Failed to apply action. Result = : [02:08:37.366] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:08:37.366] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:34099 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:08:37.366] Error starting mongod : [02:08:37.366] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34099,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 34099 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:08:36.140Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 36132 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:08:36.159+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:08:36.159+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:08:36.159+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":34099,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:08:36.160+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:08:36.163+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:08:36.164+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:08:36.183+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:08:36.183+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:08:36.183+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:08:36.228+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:08:36.228+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:08:37.866+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:08:37.866] clusterConfig unchanged [2021-10-22T02:08:37.945+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:37.945] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:38.966+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:38.966] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:39.949+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:39.949] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:40.946+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:40.946] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:41.377+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:08:41.377] memory:rss=58126336 alloc=10551656 totalalloc=9290380120 mallocs=118484813 frees=118402873 heapinuse=17367040 heapobjects=81940 gcpausetotalns=1596856436 gcpauselastns=118554 gcnum=1388 utimens=194562255000 stimens=74843721000 minflt=494314 majflt=405 [2021-10-22T02:08:41.948+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:41.948] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:42.950+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:42.950] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:43.951+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:43.950] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:44.382+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [02:08:44.382] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:34099 (local=false) because it is not up [2021-10-22T02:08:44.950+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:44.950] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:45.949+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:45.949] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:46.478+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:08:46.478] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.784189424s [2021-10-22T02:08:46.954+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:46.954] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:47.949+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:47.949] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:47.958+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:08:47.958] clusterConfig unchanged [2021-10-22T02:08:48.950+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:48.950] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:49.956+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:49.956] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:50.951+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:50.951] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:51.951+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:51.951] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:52.952+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:52.952] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:53.952+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:53.952] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:54.954+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:54.954] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:55.955+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:55.955] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:56.955+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:56.955] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:57.956+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:57.956] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:58.051+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:08:58.051] clusterConfig unchanged [2021-10-22T02:08:58.963+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:58.963] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:08:59.957+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:08:59.957] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:00.958+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:00.958] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:01.958+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:01.958] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:02.959+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:02.958] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:03.959+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:03.959] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:04.960+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:04.960] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:05.961+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:05.961] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:06.963+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:06.963] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:07.961+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:07.961] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:08.202+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:09:08.202] clusterConfig unchanged [2021-10-22T02:09:08.962+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:08.962] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:09.962+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:09.962] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:10.963+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:10.963] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:11.967+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:11.967] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:12.963+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:12.963] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:13.963+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:13.963] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:14.964+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:14.964] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:15.965+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:15.965] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:16.965+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:16.965] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:17.966+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:17.966] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:18.315+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:09:18.315] clusterConfig unchanged [2021-10-22T02:09:18.967+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:18.967] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:19.974+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:19.974] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:20.968+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:20.968] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:21.968+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:21.968] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:22.969+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:22.969] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:23.970+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:23.970] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:24.971+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:24.971] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:25.971+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:25.971] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:26.971+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:26.971] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:27.972+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:27.972] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:28.544+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:09:28.544] clusterConfig unchanged [2021-10-22T02:09:28.972+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:28.972] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:29.973+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:29.973] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:30.980+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:30.980] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:31.973+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:31.973] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:32.973+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:32.973] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:33.974+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:33.974] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:34.975+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:34.975] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:35.976+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:35.976] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:36.976+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:36.976] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:37.566+0000] [.info] [src/director/director.go:computePlan:280] [02:09:37.566] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:09:37.566+0000] [.info] [src/director/director.go:tracef:794] [02:09:37.566] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:09:37.566+0000] [.info] [src/director/director.go:tracef:794] [02:09:37.566] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:09:37.566+0000] [.info] [src/director/director.go:tracef:794] [02:09:37.566] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:09:37.566+0000] [.info] [src/director/director.go:tracef:794] [02:09:37.566] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:09:37.566+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:09:37.566] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:09:37.741+0000] [.info] [src/director/director.go:tracef:794] [02:09:37.741] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:09:37.741+0000] [.info] [src/director/director.go:tracef:794] [02:09:37.741] because [2021-10-22T02:09:37.741+0000] [.info] [src/director/director.go:tracef:794] [02:09:37.741] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:09:37.741+0000] [.info] [src/director/director.go:tracef:794] [02:09:37.741] because ['currentState.CanBounceStop' = true] [2021-10-22T02:09:37.903+0000] [.info] [src/director/director.go:tracef:794] [02:09:37.903] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:09:37.903+0000] [.info] [src/director/director.go:tracef:794] [02:09:37.903] because [2021-10-22T02:09:37.904+0000] [.info] [src/director/director.go:tracef:794] [02:09:37.904] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:09:37.904+0000] [.info] [src/director/director.go:tracef:794] [02:09:37.904] because ['currentState.Up' = false] [2021-10-22T02:09:37.904+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:09:37.904] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:09:37.904+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:09:37.904] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:09:37.977+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:37.977] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:37.995+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:09:37.995] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:09:37.995+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:09:37.995] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:09:38.003+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:09:38.003] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:09:38.004+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:09:38.004] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:09:38.005+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:09:38.005] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:09:38.005+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:09:38.005] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:09:38.005+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:09:38.005] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:09:38.007+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:09:38.007] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:09:38.237+0000] [.info] [src/director/director.go:tracef:794] [02:09:38.237] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:09:38.237+0000] [.info] [src/director/director.go:tracef:794] [02:09:38.237] because ['currentState.Up' = false] [2021-10-22T02:09:38.433+0000] [.info] [src/director/director.go:tracef:794] [02:09:38.433] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:09:38.434+0000] [.info] [src/director/director.go:tracef:794] [02:09:38.433] because ['currentState.Up' = false] [2021-10-22T02:09:38.434+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:09:38.434] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:09:38.641+0000] [.info] [src/director/director.go:tracef:794] [02:09:38.641] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:09:38.641+0000] [.info] [src/director/director.go:tracef:794] [02:09:38.641] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:09:38.774+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:09:38.774] clusterConfig unchanged [2021-10-22T02:09:38.851+0000] [.info] [src/director/director.go:tracef:794] [02:09:38.851] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:09:38.851+0000] [.info] [src/director/director.go:tracef:794] [02:09:38.851] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:09:38.851+0000] [.info] [src/director/director.go:tracef:794] [02:09:38.851] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:09:38.851+0000] [.info] [src/director/director.go:tracef:794] [02:09:38.851] because ['currentState.Up' = false] [2021-10-22T02:09:38.851+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:09:38.851] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:09:38.851+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:09:38.851] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:09:38.884+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:09:38.884] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:09:38.885+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:09:38.885] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46102 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:09:38.981+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:38.981] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:39.598+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:09:39.598] ...Finished start [pid=55313] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46102,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:09:39.740+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:09:39.740] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 141.844707ms. [2021-10-22T02:09:39.740+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:09:39.740] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46102 (local=false) [2021-10-22T02:09:39.740+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:09:39.740] Dropping existing documents from system.replset [2021-10-22T02:09:39.879+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:09:39.879] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:09:39.883+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:09:39.883] Inserting oplogTruncateAfterPoint document [2021-10-22T02:09:39.883+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:09:39.883] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:09:39.884+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:09:39.884] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:09:39.884+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:09:39.884] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:09:39.897+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:09:39.897] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:09:39.897+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:09:39.897] Setting the minvalid document to [{_id ObjectID("61721d63523370002becc79a")} {t -1} {ts {0 1}}] [2021-10-22T02:09:39.909+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:09:39.909] Inserted [{_id ObjectID("61721d63523370002becc79a")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:09:39.909+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:09:39.909] Inserting new replica set configuration document. [2021-10-22T02:09:39.909+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:09:39.909] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:09:39.917+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:09:39.917] clusterConfig unchanged [2021-10-22T02:09:39.921+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:09:39.921] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:09:39.935+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:09:39.935] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:09:39.935+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:09:39.935] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:09:39.935+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:09:39.935] Initiating stopEphemeralMongo [2021-10-22T02:09:39.935+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:09:39.935] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46102 (local=false) [2021-10-22T02:09:39.935+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:09:39.935] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46102 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:09:39.981+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:39.981] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:40.352+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:09:40.352] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.07085ms. [2021-10-22T02:09:40.352+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:09:40.352] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46102 (local=false) [2021-10-22T02:09:40.352+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:09:40.352] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.728µs. [2021-10-22T02:09:40.352+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:09:40.352] Successfully finished running stopEphemeralMongo [2021-10-22T02:09:40.352+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:09:40.352] Successfully stopped existing ephemeral mongod. [2021-10-22T02:09:40.352+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:09:40.352] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:09:40.352+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:09:40.352] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:09:40.388+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:09:40.388] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:09:40.388+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:09:40.388] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60397 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:09:40.982+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:40.982] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:41.029+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:09:41.029] clusterConfig unchanged [2021-10-22T02:09:41.386+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:09:41.386] memory:rss=56184832 alloc=14645040 totalalloc=9377931424 mallocs=119638851 frees=119514902 heapinuse=18440192 heapobjects=123949 gcpausetotalns=1598675308 gcpauselastns=835510 gcnum=1399 utimens=196611879000 stimens=75736758000 minflt=498507 majflt=405 [2021-10-22T02:09:41.645+0000] [.error] [src/action/start.go:func1:93] [103] [02:09:41.645] Error starting mongod : [02:09:41.645] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60397,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60397 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 56843 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:09:40.446+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:09:40.446+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:09:40.446+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60397,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:09:40.448+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:09:40.449+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:09:40.450+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:09:40.495+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:09:40.495+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:09:40.496+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:09:40.540+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:09:40.540+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:09:41.070+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868581:70022][56843:0x7feffe12ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:09:41.173+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868581:173551][56843:0x7feffe12ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:09:41.296+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868581:296394][56843:0x7feffe12ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35712 to 2/256"}} {"t":{"$date":"2021-10-22T02:09:41.424+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868581:424277][56843:0x7feffe12ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:09:41.497+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868581:497283][56843:0x7feffe12ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:09:41.557+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868581:557838][56843:0x7feffe12ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:09:41.557+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868581:557897][56843:0x7feffe12ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:09:41.579+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:09:41.645+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:09:41.645] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60397 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:09:41.645] Error starting mongod : [02:09:41.645] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60397,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60397 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 56843 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:09:40.446+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:09:40.446+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:09:40.446+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60397,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:09:40.448+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:09:40.449+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:09:40.450+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:09:40.495+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:09:40.495+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:09:40.496+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:09:40.540+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:09:40.540+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:09:41.070+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868581:70022][56843:0x7feffe12ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:09:41.173+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868581:173551][56843:0x7feffe12ebc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:09:41.646+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:09:41.646] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:09:41.645] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60397 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:09:41.645] Error starting mongod : [02:09:41.645] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60397,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60397 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 56843 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:09:40.446+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:09:40.446+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:09:40.446+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60397,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:09:40.448+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:09:40.449+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:09:40.450+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:09:40.495+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:09:40.495+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:09:40.496+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:09:40.540+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:09:40.540+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:09:41.070+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868581:70022][56843:0x7feffe12ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:09:41.173+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:09:41.646+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:09:41.646] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:09:41.646+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:09:41.646] Initiating stopEphemeralMongo [2021-10-22T02:09:41.646+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:09:41.646] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:09:41.647+0000] [.error] [src/director/director.go:executePlan:962] [02:09:41.647] Failed to apply action. Result = : [02:09:41.646] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:09:41.645] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60397 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:09:41.645] Error starting mongod : [02:09:41.645] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60397,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60397 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 56843 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:09:40.446+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:09:40.446+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:09:40.446+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60397,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:09:40.448+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:09:40.449+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:09:40.450+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:09:40.495+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:09:40.495+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:09:40.496+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:09:40.540+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:09:40.540+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:09:41.070+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868581:70022][56843:0x7feffe12ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:09:41.647+0000] [.error] [src/director/director.go:planAndExecute:567] [02:09:41.647] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:09:41.647] Failed to apply action. Result = : [02:09:41.646] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:09:41.645] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60397 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:09:41.645] Error starting mongod : [02:09:41.645] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60397,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60397 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 56843 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:09:40.446+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:09:40.446+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:09:40.446+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60397,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:09:40.448+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:09:40.449+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:09:40.450+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:09:40.495+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:09:40.495+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:09:40.496+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:09:40.540+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:09:40.540+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:09:41.070+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:09:41.647+0000] [.error] [src/director/director.go:mainLoop:399] [02:09:41.647] Failed to planAndExecute : [02:09:41.647] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:09:41.647] Failed to apply action. Result = : [02:09:41.646] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:09:41.645] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60397 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:09:41.645] Error starting mongod : [02:09:41.645] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60397,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60397 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 56843 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:09:40.446+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:09:40.446+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:09:40.446+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60397,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:09:40.448+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:09:40.449+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:09:40.450+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:09:40.495+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:09:40.495+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:09:40.496+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:09:40.540+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:09:40.540+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:09:41.070+00:00"},"s":"I", "c":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:09:41.983+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:41.983] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:42.289+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:09:42.289] clusterConfig unchanged [2021-10-22T02:09:42.984+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:42.984] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:43.982+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:43.982] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:44.984+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:44.984] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:45.985+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:45.985] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:46.983+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:46.983] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:47.986+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:47.986] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:48.985+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:48.985] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:49.985+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:49.985] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:50.753+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:09:50.753] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.746377782s [2021-10-22T02:09:50.985+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:50.985] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:51.985+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:51.985] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:52.392+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:09:52.392] clusterConfig unchanged [2021-10-22T02:09:52.988+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:52.988] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:53.986+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:53.986] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:54.987+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:54.987] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:55.987+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:55.987] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:56.988+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:56.988] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:57.988+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:57.988] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:58.988+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:58.988] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:09:59.995+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:09:59.995] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:00.989+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:00.989] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:01.990+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:01.990] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:02.513+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:10:02.513] clusterConfig unchanged [2021-10-22T02:10:02.991+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:02.991] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:03.991+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:03.991] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:04.992+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:04.992] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:05.993+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:05.993] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:06.994+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:06.994] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:07.995+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:07.995] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:08.994+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:08.994] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:09.997+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:09.997] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:10.995+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:10.995] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:11.996+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:11.996] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:12.615+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:10:12.615] clusterConfig unchanged [2021-10-22T02:10:12.997+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:12.997] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:13.998+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:13.998] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:14.999+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:14.999] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:15.999+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:15.999] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:16.999+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:16.999] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:18.003+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:18.003] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:19.000+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:19.000] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:20.001+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:20.001] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:21.018+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:21.018] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:22.001+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:22.001] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:22.708+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:10:22.708] clusterConfig unchanged [2021-10-22T02:10:23.002+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:23.002] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:24.002+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:24.002] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:25.002+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:25.002] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:26.003+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:26.003] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:27.003+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:27.003] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:28.005+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:28.005] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:29.005+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:29.005] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:30.005+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:30.005] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:31.006+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:31.006] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:32.009+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:32.009] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:32.838+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:10:32.838] clusterConfig unchanged [2021-10-22T02:10:33.007+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:33.007] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:34.007+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:34.007] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:35.008+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:35.008] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:36.009+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:36.009] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:37.010+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:37.010] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:38.011+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:38.011] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:39.014+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:39.014] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:40.012+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:40.012] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:41.013+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:41.013] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:41.386+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:10:41.386] memory:rss=55898112 alloc=13591504 totalalloc=9421095560 mallocs=120144877 frees=120027041 heapinuse=17432576 heapobjects=117836 gcpausetotalns=1599236948 gcpauselastns=112489 gcnum=1405 utimens=198079870000 stimens=76529195000 minflt=502160 majflt=405 [2021-10-22T02:10:41.833+0000] [.info] [src/director/director.go:computePlan:280] [02:10:41.833] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:10:41.833+0000] [.info] [src/director/director.go:tracef:794] [02:10:41.833] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:10:41.833+0000] [.info] [src/director/director.go:tracef:794] [02:10:41.833] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:10:41.833+0000] [.info] [src/director/director.go:tracef:794] [02:10:41.833] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:10:41.833+0000] [.info] [src/director/director.go:tracef:794] [02:10:41.833] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:10:41.833+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:10:41.833] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:10:41.996+0000] [.info] [src/director/director.go:tracef:794] [02:10:41.996] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:10:41.996+0000] [.info] [src/director/director.go:tracef:794] [02:10:41.996] because [2021-10-22T02:10:41.996+0000] [.info] [src/director/director.go:tracef:794] [02:10:41.996] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:10:41.996+0000] [.info] [src/director/director.go:tracef:794] [02:10:41.996] because ['currentState.CanBounceStop' = true] [2021-10-22T02:10:42.018+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:42.018] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:42.164+0000] [.info] [src/director/director.go:tracef:794] [02:10:42.164] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:10:42.164+0000] [.info] [src/director/director.go:tracef:794] [02:10:42.164] because [2021-10-22T02:10:42.164+0000] [.info] [src/director/director.go:tracef:794] [02:10:42.164] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:10:42.164+0000] [.info] [src/director/director.go:tracef:794] [02:10:42.164] because ['currentState.Up' = false] [2021-10-22T02:10:42.164+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:10:42.164] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:10:42.164+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:10:42.164] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:10:42.243+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:10:42.243] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:10:42.251+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:10:42.251] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:10:42.252+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:10:42.252] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:10:42.253+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:10:42.253] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:10:42.254+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:10:42.254] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:10:42.254+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:10:42.254] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:10:42.254+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:10:42.254] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:10:42.255+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:10:42.255] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:10:42.464+0000] [.info] [src/director/director.go:tracef:794] [02:10:42.463] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:10:42.464+0000] [.info] [src/director/director.go:tracef:794] [02:10:42.464] because ['currentState.Up' = false] [2021-10-22T02:10:42.681+0000] [.info] [src/director/director.go:tracef:794] [02:10:42.681] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:10:42.681+0000] [.info] [src/director/director.go:tracef:794] [02:10:42.681] because ['currentState.Up' = false] [2021-10-22T02:10:42.681+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:10:42.681] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:10:42.903+0000] [.info] [src/director/director.go:tracef:794] [02:10:42.903] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:10:42.903+0000] [.info] [src/director/director.go:tracef:794] [02:10:42.903] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:10:43.027+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:43.027] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:43.070+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:10:43.070] clusterConfig unchanged [2021-10-22T02:10:43.128+0000] [.info] [src/director/director.go:tracef:794] [02:10:43.128] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:10:43.128+0000] [.info] [src/director/director.go:tracef:794] [02:10:43.128] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:10:43.128+0000] [.info] [src/director/director.go:tracef:794] [02:10:43.128] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:10:43.128+0000] [.info] [src/director/director.go:tracef:794] [02:10:43.128] because ['currentState.Up' = false] [2021-10-22T02:10:43.128+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:10:43.128] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:10:43.128+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:10:43.128] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:10:43.165+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:10:43.165] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:10:43.166+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:10:43.165] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45640 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:10:43.885+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:10:43.885] ...Finished start [pid=2154] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45640,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:10:44.027+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:44.027] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:44.074+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:10:44.074] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 188.974511ms. [2021-10-22T02:10:44.074+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:10:44.074] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45640 (local=false) [2021-10-22T02:10:44.074+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:10:44.074] Dropping existing documents from system.replset [2021-10-22T02:10:44.202+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:10:44.202] clusterConfig unchanged [2021-10-22T02:10:44.281+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:10:44.281] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:10:44.304+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:10:44.304] Inserting oplogTruncateAfterPoint document [2021-10-22T02:10:44.304+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:10:44.304] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:10:44.305+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:10:44.305] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:10:44.305+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:10:44.305] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:10:44.318+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:10:44.318] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:10:44.318+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:10:44.318] Setting the minvalid document to [{_id ObjectID("61721da4523370002becc7a8")} {t -1} {ts {0 1}}] [2021-10-22T02:10:44.331+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:10:44.331] Inserted [{_id ObjectID("61721da4523370002becc7a8")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:10:44.331+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:10:44.331] Inserting new replica set configuration document. [2021-10-22T02:10:44.331+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:10:44.331] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:10:44.344+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:10:44.344] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:10:44.348+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:10:44.348] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:10:44.348+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:10:44.348] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:10:44.348+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:10:44.348] Initiating stopEphemeralMongo [2021-10-22T02:10:44.348+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:10:44.348] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45640 (local=false) [2021-10-22T02:10:44.348+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:10:44.348] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45640 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:10:44.708+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:10:44.707] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.716201ms. [2021-10-22T02:10:44.708+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:10:44.708] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45640 (local=false) [2021-10-22T02:10:44.708+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:10:44.708] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 55.287µs. [2021-10-22T02:10:44.708+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:10:44.708] Successfully finished running stopEphemeralMongo [2021-10-22T02:10:44.708+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:10:44.708] Successfully stopped existing ephemeral mongod. [2021-10-22T02:10:44.708+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:10:44.708] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:10:44.708+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:10:44.708] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:10:44.755+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:10:44.755] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:10:44.757+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:10:44.757] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51787 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:10:45.029+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:45.029] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:45.315+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:10:45.315] clusterConfig unchanged [2021-10-22T02:10:45.924+0000] [.error] [src/action/start.go:func1:93] [103] [02:10:45.924] Error starting mongod : [02:10:45.923] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51787,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51787 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 3774 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:10:44.813+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:10:44.813+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:10:44.813+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51787,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:10:44.814+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:10:44.816+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:10:44.827+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:10:44.862+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:10:44.863+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:10:44.864+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:10:44.908+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:10:44.908+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:10:45.422+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868645:422500][3774:0x7fbac92cfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:10:45.526+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868645:526137][3774:0x7fbac92cfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:10:45.628+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868645:628494][3774:0x7fbac92cfbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T02:10:45.743+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868645:743155][3774:0x7fbac92cfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:10:45.816+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868645:816394][3774:0x7fbac92cfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:10:45.874+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868645:874778][3774:0x7fbac92cfbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:10:45.874+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868645:874852][3774:0x7fbac92cfbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:10:45.876+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:10:45.924+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:10:45.924] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51787 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:10:45.924] Error starting mongod : [02:10:45.923] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51787,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51787 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 3774 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:10:44.813+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:10:44.813+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:10:44.813+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51787,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:10:44.814+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:10:44.816+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:10:44.827+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:10:44.862+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:10:44.863+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:10:44.864+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:10:44.908+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:10:44.908+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:10:45.422+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868645:422500][3774:0x7fbac92cfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:10:45.526+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868645:526137][3774:0x7fbac92cfbc0], txn-recover...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:10:45.924+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:10:45.924] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:10:45.924] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51787 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:10:45.924] Error starting mongod : [02:10:45.923] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51787,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51787 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 3774 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:10:44.813+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:10:44.813+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:10:44.813+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51787,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:10:44.814+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:10:44.816+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:10:44.827+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:10:44.862+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:10:44.863+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:10:44.864+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:10:44.908+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:10:44.908+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:10:45.422+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868645:422500][3774:0x7fbac92cfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:10:45.526+00:00"},"s":"I", "c":"STORAGE", "id":22430, ... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:10:45.924+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:10:45.924] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:10:45.924+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:10:45.924] Initiating stopEphemeralMongo [2021-10-22T02:10:45.924+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:10:45.924] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:10:45.926+0000] [.error] [src/director/director.go:executePlan:962] [02:10:45.926] Failed to apply action. Result = : [02:10:45.924] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:10:45.924] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51787 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:10:45.924] Error starting mongod : [02:10:45.923] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51787,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51787 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 3774 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:10:44.813+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:10:44.813+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:10:44.813+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51787,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:10:44.814+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:10:44.816+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:10:44.827+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:10:44.862+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:10:44.863+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:10:44.864+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:10:44.908+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:10:44.908+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:10:45.422+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868645:422500][3774:0x7fbac92cfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:10:45.926+0000] [.error] [src/director/director.go:planAndExecute:567] [02:10:45.926] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:10:45.926] Failed to apply action. Result = : [02:10:45.924] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:10:45.924] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51787 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:10:45.924] Error starting mongod : [02:10:45.923] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51787,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51787 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 3774 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:10:44.813+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:10:44.813+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:10:44.813+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51787,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:10:44.814+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:10:44.816+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:10:44.827+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:10:44.862+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:10:44.863+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:10:44.864+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:10:44.908+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:10:44.908+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:10:45.422+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger m... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:10:45.926+0000] [.error] [src/director/director.go:mainLoop:399] [02:10:45.926] Failed to planAndExecute : [02:10:45.926] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:10:45.926] Failed to apply action. Result = : [02:10:45.924] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:10:45.924] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51787 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:10:45.924] Error starting mongod : [02:10:45.923] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51787,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51787 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 3774 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:10:44.813+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:10:44.813+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:10:44.813+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51787,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:10:44.814+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:10:44.816+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:10:44.827+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:10:44.862+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:10:44.863+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:10:44.864+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:10:44.908+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:10:44.908+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:10:45.422+00:00"},"s":"I",... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:10:46.029+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:46.029] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:46.605+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:10:46.605] clusterConfig unchanged [2021-10-22T02:10:47.033+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:47.033] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:48.030+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:48.030] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:49.031+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:49.031] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:50.030+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:50.030] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:51.030+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:51.030] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:52.051+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:52.051] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:53.031+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:53.031] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:54.033+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:54.033] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:54.971+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:10:54.971] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.71638421s [2021-10-22T02:10:55.032+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:55.032] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:56.032+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:56.032] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:56.736+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:10:56.736] clusterConfig unchanged [2021-10-22T02:10:57.033+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:57.033] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:58.033+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:58.033] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:10:59.034+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:10:59.034] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:00.035+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:00.035] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:01.036+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:01.036] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:02.037+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:02.037] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:03.043+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:03.043] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:04.037+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:04.037] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:05.037+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:05.037] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:06.039+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:06.039] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:06.868+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:11:06.868] clusterConfig unchanged [2021-10-22T02:11:07.038+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:07.038] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:08.038+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:08.038] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:09.039+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:09.039] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:10.039+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:10.039] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:11.040+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:11.040] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:12.042+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:12.042] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:13.040+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:13.040] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:14.041+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:14.041] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:15.049+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:15.049] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:16.042+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:16.042] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:17.042+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:17.042] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:17.139+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:11:17.139] clusterConfig unchanged [2021-10-22T02:11:18.043+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:18.043] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:18.565+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:11:18.565] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:11:18.567+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:18.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:11:18.567+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [02:11:18.567] Could not get version : [02:11:18.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:11:18.567+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [02:11:18.567] Error getting version for log lines. Skipping : [02:11:18.567] Could not get version : [02:11:18.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:11:19.044+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:19.044] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:20.045+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:20.045] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:21.045+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:21.045] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:22.046+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:22.046] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:23.048+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:23.048] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:24.049+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:24.049] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:25.047+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:25.047] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:26.049+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:26.049] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:27.049+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:27.049] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:27.249+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:11:27.249] clusterConfig unchanged [2021-10-22T02:11:28.048+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:28.048] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:29.050+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:29.050] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:30.049+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:30.049] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:31.050+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:31.050] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:32.051+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:32.051] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:33.050+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:33.050] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:34.051+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:34.051] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:35.052+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:35.052] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:36.051+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:36.051] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:37.051+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:37.051] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:37.361+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:11:37.361] clusterConfig unchanged [2021-10-22T02:11:38.051+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:38.051] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:39.052+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:39.052] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:40.053+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:40.053] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:41.054+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:41.054] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:41.387+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:11:41.387] memory:rss=55488512 alloc=11740080 totalalloc=9510823512 mallocs=121303465 frees=121207264 heapinuse=16867328 heapobjects=96201 gcpausetotalns=1600370228 gcpauselastns=104947 gcnum=1417 utimens=200378267000 stimens=77547356000 minflt=507597 majflt=405 [2021-10-22T02:11:42.055+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:42.055] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:43.055+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:43.055] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:44.062+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:44.062] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:44.391+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [02:11:44.391] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51787 (local=false) because it is not up [2021-10-22T02:11:45.055+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:45.055] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:46.059+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:46.059] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:46.097+0000] [.info] [src/director/director.go:computePlan:280] [02:11:46.097] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:11:46.097+0000] [.info] [src/director/director.go:tracef:794] [02:11:46.097] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:11:46.097+0000] [.info] [src/director/director.go:tracef:794] [02:11:46.097] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:11:46.097+0000] [.info] [src/director/director.go:tracef:794] [02:11:46.097] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:11:46.097+0000] [.info] [src/director/director.go:tracef:794] [02:11:46.097] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:11:46.097+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:11:46.097] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:11:46.250+0000] [.info] [src/director/director.go:tracef:794] [02:11:46.250] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:11:46.250+0000] [.info] [src/director/director.go:tracef:794] [02:11:46.250] because [2021-10-22T02:11:46.250+0000] [.info] [src/director/director.go:tracef:794] [02:11:46.250] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:11:46.250+0000] [.info] [src/director/director.go:tracef:794] [02:11:46.250] because ['currentState.CanBounceStop' = true] [2021-10-22T02:11:46.403+0000] [.info] [src/director/director.go:tracef:794] [02:11:46.403] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:11:46.403+0000] [.info] [src/director/director.go:tracef:794] [02:11:46.403] because [2021-10-22T02:11:46.403+0000] [.info] [src/director/director.go:tracef:794] [02:11:46.403] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:11:46.403+0000] [.info] [src/director/director.go:tracef:794] [02:11:46.403] because ['currentState.Up' = false] [2021-10-22T02:11:46.403+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:11:46.403] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:11:46.404+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:11:46.403] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:11:46.488+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:11:46.488] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:11:46.496+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:11:46.496] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:11:46.496+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:11:46.496] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:11:46.497+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:11:46.497] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:11:46.498+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:11:46.498] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:11:46.498+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:11:46.498] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:11:46.498+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:11:46.498] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:11:46.500+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:11:46.500] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:11:46.703+0000] [.info] [src/director/director.go:tracef:794] [02:11:46.703] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:11:46.703+0000] [.info] [src/director/director.go:tracef:794] [02:11:46.703] because ['currentState.Up' = false] [2021-10-22T02:11:46.906+0000] [.info] [src/director/director.go:tracef:794] [02:11:46.906] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:11:46.906+0000] [.info] [src/director/director.go:tracef:794] [02:11:46.906] because ['currentState.Up' = false] [2021-10-22T02:11:46.906+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:11:46.906] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:11:47.058+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:47.058] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:47.110+0000] [.info] [src/director/director.go:tracef:794] [02:11:47.110] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:11:47.110+0000] [.info] [src/director/director.go:tracef:794] [02:11:47.110] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:11:47.330+0000] [.info] [src/director/director.go:tracef:794] [02:11:47.330] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:11:47.330+0000] [.info] [src/director/director.go:tracef:794] [02:11:47.330] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:11:47.330+0000] [.info] [src/director/director.go:tracef:794] [02:11:47.330] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:11:47.331+0000] [.info] [src/director/director.go:tracef:794] [02:11:47.330] because ['currentState.Up' = false] [2021-10-22T02:11:47.331+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:11:47.331] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:11:47.331+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:11:47.331] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:11:47.366+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:11:47.366] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:11:47.367+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:11:47.367] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 53033 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:11:47.554+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:11:47.554] clusterConfig unchanged [2021-10-22T02:11:48.058+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:48.058] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:48.099+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:11:48.099] ...Finished start [pid=22792] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":53033,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:11:48.221+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:11:48.221] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 121.989638ms. [2021-10-22T02:11:48.221+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:11:48.221] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53033 (local=false) [2021-10-22T02:11:48.221+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:11:48.221] Dropping existing documents from system.replset [2021-10-22T02:11:48.354+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:11:48.354] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:11:48.360+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:11:48.360] Inserting oplogTruncateAfterPoint document [2021-10-22T02:11:48.360+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:11:48.360] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:11:48.362+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:11:48.361] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:11:48.362+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:11:48.362] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:11:48.375+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:11:48.375] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:11:48.375+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:11:48.375] Setting the minvalid document to [{_id ObjectID("61721de4523370002becc7b6")} {t -1} {ts {0 1}}] [2021-10-22T02:11:48.387+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:11:48.387] Inserted [{_id ObjectID("61721de4523370002becc7b6")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:11:48.387+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:11:48.387] Inserting new replica set configuration document. [2021-10-22T02:11:48.387+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:11:48.387] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:11:48.399+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:11:48.399] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:11:48.403+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:11:48.403] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:11:48.403+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:11:48.403] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:11:48.403+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:11:48.403] Initiating stopEphemeralMongo [2021-10-22T02:11:48.403+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:11:48.403] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53033 (local=false) [2021-10-22T02:11:48.403+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:11:48.403] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53033 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:11:48.676+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:11:48.676] clusterConfig unchanged [2021-10-22T02:11:48.752+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:11:48.752] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.9297ms. [2021-10-22T02:11:48.752+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:11:48.752] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:53033 (local=false) [2021-10-22T02:11:48.752+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:11:48.752] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.285µs. [2021-10-22T02:11:48.752+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:11:48.752] Successfully finished running stopEphemeralMongo [2021-10-22T02:11:48.752+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:11:48.752] Successfully stopped existing ephemeral mongod. [2021-10-22T02:11:48.752+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:11:48.752] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:11:48.752+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:11:48.752] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:11:48.785+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:11:48.785] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:11:48.786+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:11:48.786] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47717 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:11:49.060+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:49.060] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:49.767+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:11:49.767] clusterConfig unchanged [2021-10-22T02:11:49.989+0000] [.error] [src/action/start.go:func1:93] [103] [02:11:49.989] Error starting mongod : [02:11:49.989] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47717,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47717 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 24273 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:11:48.848+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:11:48.848+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:11:48.848+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47717,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:11:48.850+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:11:48.853+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:11:48.853+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:11:48.865+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:11:48.865+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:11:48.865+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:11:48.910+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:11:48.910+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:11:49.454+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868709:454354][24273:0x7f340e8b4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:11:49.558+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868709:558186][24273:0x7f340e8b4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:11:49.660+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868709:660051][24273:0x7f340e8b4bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:11:49.773+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868709:773977][24273:0x7f340e8b4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:11:49.849+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868709:849285][24273:0x7f340e8b4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:11:49.915+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868709:915761][24273:0x7f340e8b4bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:11:49.915+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868709:915852][24273:0x7f340e8b4bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:11:49.917+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:11:49.989+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:11:49.989] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:47717 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:11:49.989] Error starting mongod : [02:11:49.989] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47717,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47717 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 24273 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:11:48.848+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:11:48.848+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:11:48.848+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47717,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:11:48.850+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:11:48.853+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:11:48.853+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:11:48.865+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:11:48.865+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:11:48.865+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:11:48.910+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:11:48.910+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:11:49.454+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868709:454354][24273:0x7f340e8b4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:11:49.558+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868709:558186][24273:0x7f340e8b4bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:11:49.989+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:11:49.989] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:11:49.989] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:47717 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:11:49.989] Error starting mongod : [02:11:49.989] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47717,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47717 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 24273 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:11:48.848+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:11:48.848+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:11:48.848+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47717,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:11:48.850+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:11:48.853+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:11:48.853+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:11:48.865+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:11:48.865+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:11:48.865+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:11:48.910+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:11:48.910+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:11:49.454+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868709:454354][24273:0x7f340e8b4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:11:49.558+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:11:49.990+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:11:49.990] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:11:49.990+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:11:49.990] Initiating stopEphemeralMongo [2021-10-22T02:11:49.990+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:11:49.990] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:11:49.990+0000] [.error] [src/director/director.go:executePlan:962] [02:11:49.990] Failed to apply action. Result = : [02:11:49.989] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:11:49.989] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:47717 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:11:49.989] Error starting mongod : [02:11:49.989] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47717,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47717 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 24273 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:11:48.848+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:11:48.848+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:11:48.848+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47717,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:11:48.850+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:11:48.853+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:11:48.853+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:11:48.865+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:11:48.865+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:11:48.865+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:11:48.910+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:11:48.910+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:11:49.454+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868709:454354][24273:0x7f340e8b4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:11:49.990+0000] [.error] [src/director/director.go:planAndExecute:567] [02:11:49.990] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:11:49.990] Failed to apply action. Result = : [02:11:49.989] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:11:49.989] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:47717 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:11:49.989] Error starting mongod : [02:11:49.989] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47717,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47717 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 24273 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:11:48.848+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:11:48.848+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:11:48.848+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47717,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:11:48.850+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:11:48.853+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:11:48.853+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:11:48.865+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:11:48.865+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:11:48.865+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:11:48.910+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:11:48.910+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:11:49.454+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:11:49.990+0000] [.error] [src/director/director.go:mainLoop:399] [02:11:49.990] Failed to planAndExecute : [02:11:49.990] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:11:49.990] Failed to apply action. Result = : [02:11:49.989] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:11:49.989] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:47717 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:11:49.989] Error starting mongod : [02:11:49.989] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47717,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47717 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 24273 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:11:48.848+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:11:48.848+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:11:48.848+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47717,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:11:48.850+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:11:48.853+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:11:48.853+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:11:48.865+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:11:48.865+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:11:48.865+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:11:48.910+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:11:48.910+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:11:49.454+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:11:50.060+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:50.060] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:51.027+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:11:51.027] clusterConfig unchanged [2021-10-22T02:11:51.067+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:51.067] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:52.083+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:52.083] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:53.066+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:53.066] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:54.068+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:54.068] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:55.068+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:55.068] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:56.076+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:56.076] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:57.075+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:57.075] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:58.077+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:58.077] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:59.077+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:11:59.077] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:11:59.136+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:11:59.136] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.636060679s [2021-10-22T02:12:00.074+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:00.074] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:01.074+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:01.074] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:01.163+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:12:01.163] clusterConfig unchanged [2021-10-22T02:12:02.079+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:02.079] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:03.076+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:03.076] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:04.078+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:04.078] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:05.077+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:05.077] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:06.081+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:06.081] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:07.078+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:07.078] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:08.078+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:08.078] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:09.079+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:09.079] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:10.079+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:10.079] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:11.079+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:11.079] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:11.266+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:12:11.266] clusterConfig unchanged [2021-10-22T02:12:12.080+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:12.080] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:13.081+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:13.081] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:14.082+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:14.082] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:15.082+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:15.082] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:16.083+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:16.083] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:17.083+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:17.083] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:18.084+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:18.084] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:19.085+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:19.085] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:20.085+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:20.085] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:21.086+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:21.086] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:21.407+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:12:21.407] clusterConfig unchanged [2021-10-22T02:12:22.087+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:22.087] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:23.087+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:23.087] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:24.087+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:24.087] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:25.095+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:25.095] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:26.088+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:26.088] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:27.088+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:27.088] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:27.569+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:12:27.569] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:12:27.570+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:12:27.570] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:12:28.088+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:28.088] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:29.089+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:29.089] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:30.090+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:30.090] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:31.091+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:31.091] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:31.509+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:12:31.509] clusterConfig unchanged [2021-10-22T02:12:32.091+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:32.091] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:33.091+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:33.091] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:34.092+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:34.092] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:35.092+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:35.092] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:36.093+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:36.093] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:37.100+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:37.100] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:38.095+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [02:12:38.095] lsblk cache invalidated, running lsblk command [2021-10-22T02:12:38.096+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:38.096] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:39.095+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:39.095] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:40.094+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:40.094] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:41.094+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:41.094] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:41.388+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:12:41.388] memory:rss=56922112 alloc=11077248 totalalloc=9603311112 mallocs=122483142 frees=122395746 heapinuse=17211392 heapobjects=87396 gcpausetotalns=1601725810 gcpauselastns=65538 gcnum=1429 utimens=202672911000 stimens=78531954000 minflt=512746 majflt=405 [2021-10-22T02:12:41.612+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:12:41.612] clusterConfig unchanged [2021-10-22T02:12:42.094+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:42.094] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:43.095+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:43.095] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:44.099+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:44.099] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:45.097+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:45.097] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:46.102+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:46.102] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:47.097+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:47.097] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:48.098+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:48.098] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:49.098+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:49.098] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:50.118+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:50.118] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:50.165+0000] [.info] [src/director/director.go:computePlan:280] [02:12:50.165] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:12:50.165+0000] [.info] [src/director/director.go:tracef:794] [02:12:50.165] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:12:50.165+0000] [.info] [src/director/director.go:tracef:794] [02:12:50.165] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:12:50.166+0000] [.info] [src/director/director.go:tracef:794] [02:12:50.166] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:12:50.166+0000] [.info] [src/director/director.go:tracef:794] [02:12:50.166] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:12:50.166+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:12:50.166] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:12:50.330+0000] [.info] [src/director/director.go:tracef:794] [02:12:50.330] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:12:50.330+0000] [.info] [src/director/director.go:tracef:794] [02:12:50.330] because [2021-10-22T02:12:50.330+0000] [.info] [src/director/director.go:tracef:794] [02:12:50.330] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:12:50.330+0000] [.info] [src/director/director.go:tracef:794] [02:12:50.330] because ['currentState.CanBounceStop' = true] [2021-10-22T02:12:50.483+0000] [.info] [src/director/director.go:tracef:794] [02:12:50.483] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:12:50.483+0000] [.info] [src/director/director.go:tracef:794] [02:12:50.483] because [2021-10-22T02:12:50.483+0000] [.info] [src/director/director.go:tracef:794] [02:12:50.483] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:12:50.483+0000] [.info] [src/director/director.go:tracef:794] [02:12:50.483] because ['currentState.Up' = false] [2021-10-22T02:12:50.483+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:12:50.483] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:12:50.483+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:12:50.483] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:12:50.562+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:12:50.562] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:12:50.570+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:12:50.570] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:12:50.571+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:12:50.571] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:12:50.572+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:12:50.572] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:12:50.572+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:12:50.572] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:12:50.572+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:12:50.572] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:12:50.574+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:12:50.574] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:12:50.802+0000] [.info] [src/director/director.go:tracef:794] [02:12:50.802] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:12:50.802+0000] [.info] [src/director/director.go:tracef:794] [02:12:50.802] because ['currentState.Up' = false] [2021-10-22T02:12:51.033+0000] [.info] [src/director/director.go:tracef:794] [02:12:51.033] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:12:51.033+0000] [.info] [src/director/director.go:tracef:794] [02:12:51.033] because ['currentState.Up' = false] [2021-10-22T02:12:51.033+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:12:51.033] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:12:51.108+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:51.108] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:51.243+0000] [.info] [src/director/director.go:tracef:794] [02:12:51.243] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:12:51.243+0000] [.info] [src/director/director.go:tracef:794] [02:12:51.243] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:12:51.455+0000] [.info] [src/director/director.go:tracef:794] [02:12:51.455] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:12:51.455+0000] [.info] [src/director/director.go:tracef:794] [02:12:51.455] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:12:51.455+0000] [.info] [src/director/director.go:tracef:794] [02:12:51.455] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:12:51.455+0000] [.info] [src/director/director.go:tracef:794] [02:12:51.455] because ['currentState.Up' = false] [2021-10-22T02:12:51.455+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:12:51.455] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:12:51.455+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:12:51.455] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:12:51.491+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:12:51.491] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:12:51.492+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:12:51.492] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50707 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:12:51.803+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:12:51.803] clusterConfig unchanged [2021-10-22T02:12:52.103+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:52.103] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:52.230+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:12:52.230] ...Finished start [pid=43640] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50707,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:12:52.354+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:12:52.353] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 123.750729ms. [2021-10-22T02:12:52.354+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:12:52.354] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50707 (local=false) [2021-10-22T02:12:52.354+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:12:52.354] Dropping existing documents from system.replset [2021-10-22T02:12:52.513+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:12:52.513] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:12:52.517+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:12:52.517] Inserting oplogTruncateAfterPoint document [2021-10-22T02:12:52.517+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:12:52.517] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:12:52.518+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:12:52.518] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:12:52.518+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:12:52.518] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:12:52.532+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:12:52.532] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:12:52.532+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:12:52.532] Setting the minvalid document to [{_id ObjectID("61721e24523370002becc7c4")} {t -1} {ts {0 1}}] [2021-10-22T02:12:52.546+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:12:52.546] Inserted [{_id ObjectID("61721e24523370002becc7c4")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:12:52.546+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:12:52.546] Inserting new replica set configuration document. [2021-10-22T02:12:52.546+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:12:52.546] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:12:52.559+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:12:52.559] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:12:52.573+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:12:52.573] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:12:52.573+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:12:52.573] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:12:52.573+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:12:52.573] Initiating stopEphemeralMongo [2021-10-22T02:12:52.573+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:12:52.573] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50707 (local=false) [2021-10-22T02:12:52.573+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:12:52.573] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50707 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:12:52.895+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:12:52.895] clusterConfig unchanged [2021-10-22T02:12:52.947+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:12:52.947] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.901491ms. [2021-10-22T02:12:52.947+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:12:52.947] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50707 (local=false) [2021-10-22T02:12:52.947+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:12:52.947] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.577µs. [2021-10-22T02:12:52.947+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:12:52.947] Successfully finished running stopEphemeralMongo [2021-10-22T02:12:52.947+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:12:52.947] Successfully stopped existing ephemeral mongod. [2021-10-22T02:12:52.947+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:12:52.947] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:12:52.947+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:12:52.947] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:12:52.981+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:12:52.981] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:12:52.982+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:12:52.982] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42796 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:12:53.104+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:53.104] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:53.999+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:12:53.999] clusterConfig unchanged [2021-10-22T02:12:54.111+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:54.111] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:54.200+0000] [.error] [src/action/start.go:func1:93] [103] [02:12:54.200] Error starting mongod : [02:12:54.200] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42796,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42796 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 45119 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:12:53.033+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:12:53.033+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:12:53.034+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42796,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:12:53.035+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:12:53.037+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:12:53.038+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:12:53.105+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:12:53.105+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:12:53.106+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:12:53.150+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:12:53.150+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:12:53.647+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868773:647482][45119:0x7f9de3f58bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:12:53.747+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868773:747181][45119:0x7f9de3f58bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:12:53.885+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868773:885860][45119:0x7f9de3f58bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:12:54.008+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868774:8818][45119:0x7f9de3f58bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:12:54.086+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868774:86147][45119:0x7f9de3f58bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:12:54.150+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868774:150738][45119:0x7f9de3f58bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:12:54.150+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868774:150808][45119:0x7f9de3f58bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:12:54.152+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:12:54.200+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:12:54.200] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42796 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:12:54.200] Error starting mongod : [02:12:54.200] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42796,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42796 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 45119 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:12:53.033+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:12:53.033+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:12:53.034+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42796,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:12:53.035+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:12:53.037+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:12:53.038+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:12:53.105+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:12:53.105+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:12:53.106+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:12:53.150+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:12:53.150+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:12:53.647+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868773:647482][45119:0x7f9de3f58bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:12:53.747+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868773:747181][45119:0x7f9de3f58bc0], txn-recover: ...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:12:54.201+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:12:54.201] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:12:54.200] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42796 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:12:54.200] Error starting mongod : [02:12:54.200] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42796,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42796 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 45119 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:12:53.033+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:12:53.033+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:12:53.034+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42796,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:12:53.035+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:12:53.037+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:12:53.038+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:12:53.105+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:12:53.105+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:12:53.106+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:12:53.150+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:12:53.150+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:12:53.647+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868773:647482][45119:0x7f9de3f58bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:12:53.747+00:00"},"s":"I", "c":"STORAGE", "id":22430, "... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:12:54.201+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:12:54.201] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:12:54.201+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:12:54.201] Initiating stopEphemeralMongo [2021-10-22T02:12:54.201+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:12:54.201] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:12:54.203+0000] [.error] [src/director/director.go:executePlan:962] [02:12:54.202] Failed to apply action. Result = : [02:12:54.201] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:12:54.200] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42796 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:12:54.200] Error starting mongod : [02:12:54.200] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42796,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42796 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 45119 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:12:53.033+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:12:53.033+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:12:53.034+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42796,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:12:53.035+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:12:53.037+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:12:53.038+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:12:53.105+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:12:53.105+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:12:53.106+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:12:53.150+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:12:53.150+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:12:53.647+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868773:647482][45119:0x7f9de3f58bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:12:54.203+0000] [.error] [src/director/director.go:planAndExecute:567] [02:12:54.203] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:12:54.202] Failed to apply action. Result = : [02:12:54.201] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:12:54.200] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42796 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:12:54.200] Error starting mongod : [02:12:54.200] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42796,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42796 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 45119 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:12:53.033+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:12:53.033+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:12:53.034+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42796,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:12:53.035+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:12:53.037+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:12:53.038+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:12:53.105+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:12:53.105+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:12:53.106+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:12:53.150+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:12:53.150+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:12:53.647+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger messa... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:12:54.205+0000] [.error] [src/director/director.go:mainLoop:399] [02:12:54.204] Failed to planAndExecute : [02:12:54.203] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:12:54.202] Failed to apply action. Result = : [02:12:54.201] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:12:54.200] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42796 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:12:54.200] Error starting mongod : [02:12:54.200] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42796,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42796 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 45119 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:12:53.033+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:12:53.033+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:12:53.034+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42796,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:12:53.035+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:12:53.037+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:12:53.038+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:12:53.105+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:12:53.105+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:12:53.106+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:12:53.150+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:12:53.150+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:12:53.647+00:00"},"s":"I", "c... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:12:55.110+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:55.110] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:55.258+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:12:55.258] clusterConfig unchanged [2021-10-22T02:12:56.111+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:56.111] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:57.112+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:57.112] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:58.111+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:58.111] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:12:59.112+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:12:59.112] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:00.114+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:00.114] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:01.112+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:01.112] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:02.114+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:02.114] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:03.114+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:03.114] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:03.406+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:13:03.406] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.832033208s [2021-10-22T02:13:04.112+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:04.112] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:05.112+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:05.112] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:05.360+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:13:05.360] clusterConfig unchanged [2021-10-22T02:13:06.113+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:06.113] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:07.121+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:07.121] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:08.115+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:08.115] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:09.115+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:09.115] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:10.116+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:10.116] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:11.116+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:11.116] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:12.116+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:12.116] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:13.116+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:13.116] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:14.117+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:14.117] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:15.118+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:15.118] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:15.484+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:13:15.484] clusterConfig unchanged [2021-10-22T02:13:16.118+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:16.118] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:17.118+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:17.118] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:18.118+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:18.118] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:19.118+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:19.118] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:20.119+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:20.119] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:21.120+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:21.120] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:22.121+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:22.121] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:23.122+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:23.122] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:24.130+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:24.130] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:25.123+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:25.123] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:25.604+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:13:25.604] clusterConfig unchanged [2021-10-22T02:13:26.124+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:26.124] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:27.125+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:27.125] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:28.127+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:28.127] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:29.126+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:29.126] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:30.126+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:30.126] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:31.126+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:31.126] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:32.127+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:32.127] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:33.128+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:33.128] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:34.129+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:34.129] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:35.129+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:35.129] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:35.716+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:13:35.716] clusterConfig unchanged [2021-10-22T02:13:36.130+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:36.130] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:37.131+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:37.131] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:38.132+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:38.132] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:39.133+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:39.133] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:40.133+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:40.133] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:41.135+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:41.135] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:41.388+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:13:41.388] memory:rss=55967744 alloc=11695200 totalalloc=9694899376 mallocs=123655014 frees=123559383 heapinuse=17080320 heapobjects=95631 gcpausetotalns=1602891990 gcpauselastns=87680 gcnum=1441 utimens=204978419000 stimens=79567728000 minflt=517819 majflt=405 [2021-10-22T02:13:42.136+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:42.136] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:43.136+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:43.136] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:44.142+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:44.142] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:45.139+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:45.138] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:45.821+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:13:45.821] clusterConfig unchanged [2021-10-22T02:13:46.139+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:46.138] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:47.139+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:47.139] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:48.139+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:48.139] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:49.140+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:49.140] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:50.140+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:50.140] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:51.141+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:51.141] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:52.141+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:52.141] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:53.142+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:53.141] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:54.142+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:54.142] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:54.381+0000] [.info] [src/director/director.go:computePlan:280] [02:13:54.381] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:13:54.381+0000] [.info] [src/director/director.go:tracef:794] [02:13:54.381] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:13:54.381+0000] [.info] [src/director/director.go:tracef:794] [02:13:54.381] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:13:54.381+0000] [.info] [src/director/director.go:tracef:794] [02:13:54.381] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:13:54.381+0000] [.info] [src/director/director.go:tracef:794] [02:13:54.381] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:13:54.381+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:13:54.381] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:13:54.528+0000] [.info] [src/director/director.go:tracef:794] [02:13:54.528] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:13:54.528+0000] [.info] [src/director/director.go:tracef:794] [02:13:54.528] because [2021-10-22T02:13:54.529+0000] [.info] [src/director/director.go:tracef:794] [02:13:54.529] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:13:54.529+0000] [.info] [src/director/director.go:tracef:794] [02:13:54.529] because ['currentState.CanBounceStop' = true] [2021-10-22T02:13:54.685+0000] [.info] [src/director/director.go:tracef:794] [02:13:54.685] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:13:54.685+0000] [.info] [src/director/director.go:tracef:794] [02:13:54.685] because [2021-10-22T02:13:54.685+0000] [.info] [src/director/director.go:tracef:794] [02:13:54.685] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:13:54.685+0000] [.info] [src/director/director.go:tracef:794] [02:13:54.685] because ['currentState.Up' = false] [2021-10-22T02:13:54.685+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:13:54.685] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:13:54.685+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:13:54.685] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:13:54.769+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:13:54.769] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:13:54.777+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:13:54.777] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:13:54.778+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:13:54.778] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:13:54.779+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:13:54.779] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:13:54.779+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:13:54.779] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:13:54.779+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:13:54.779] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:13:54.780+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:13:54.780] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:13:54.985+0000] [.info] [src/director/director.go:tracef:794] [02:13:54.985] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:13:54.985+0000] [.info] [src/director/director.go:tracef:794] [02:13:54.985] because ['currentState.Up' = false] [2021-10-22T02:13:55.162+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:55.162] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:55.211+0000] [.info] [src/director/director.go:tracef:794] [02:13:55.211] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:13:55.211+0000] [.info] [src/director/director.go:tracef:794] [02:13:55.211] because ['currentState.Up' = false] [2021-10-22T02:13:55.211+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:13:55.211] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:13:55.441+0000] [.info] [src/director/director.go:tracef:794] [02:13:55.441] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:13:55.441+0000] [.info] [src/director/director.go:tracef:794] [02:13:55.441] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:13:55.657+0000] [.info] [src/director/director.go:tracef:794] [02:13:55.657] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:13:55.657+0000] [.info] [src/director/director.go:tracef:794] [02:13:55.657] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:13:55.657+0000] [.info] [src/director/director.go:tracef:794] [02:13:55.657] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:13:55.657+0000] [.info] [src/director/director.go:tracef:794] [02:13:55.657] because ['currentState.Up' = false] [2021-10-22T02:13:55.657+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:13:55.657] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:13:55.657+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:13:55.657] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:13:55.694+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:13:55.694] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:13:55.694+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:13:55.694] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36656 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:13:56.040+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:13:56.040] clusterConfig unchanged [2021-10-22T02:13:56.157+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:56.157] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:56.420+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:13:56.420] ...Finished start [pid=64508] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36656,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:13:56.573+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:13:56.573] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 152.718537ms. [2021-10-22T02:13:56.573+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:13:56.573] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36656 (local=false) [2021-10-22T02:13:56.573+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:13:56.573] Dropping existing documents from system.replset [2021-10-22T02:13:56.785+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:13:56.785] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:13:56.790+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:13:56.790] Inserting oplogTruncateAfterPoint document [2021-10-22T02:13:56.790+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:13:56.790] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:13:56.791+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:13:56.791] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:13:56.791+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:13:56.791] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:13:56.805+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:13:56.804] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:13:56.805+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:13:56.805] Setting the minvalid document to [{_id ObjectID("61721e64523370002becc7d2")} {t -1} {ts {0 1}}] [2021-10-22T02:13:56.820+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:13:56.820] Inserted [{_id ObjectID("61721e64523370002becc7d2")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:13:56.820+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:13:56.820] Inserting new replica set configuration document. [2021-10-22T02:13:56.820+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:13:56.820] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:13:56.834+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:13:56.834] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:13:56.848+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:13:56.848] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:13:56.848+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:13:56.848] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:13:56.848+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:13:56.848] Initiating stopEphemeralMongo [2021-10-22T02:13:56.848+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:13:56.848] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36656 (local=false) [2021-10-22T02:13:56.848+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:13:56.848] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36656 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:13:57.162+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:13:57.162] clusterConfig unchanged [2021-10-22T02:13:57.164+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:57.164] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:57.279+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:13:57.279] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 3.85569ms. [2021-10-22T02:13:57.279+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:13:57.279] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:36656 (local=false) [2021-10-22T02:13:57.279+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:13:57.279] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 42.109µs. [2021-10-22T02:13:57.280+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:13:57.280] Successfully finished running stopEphemeralMongo [2021-10-22T02:13:57.280+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:13:57.280] Successfully stopped existing ephemeral mongod. [2021-10-22T02:13:57.280+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:13:57.280] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:13:57.280+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:13:57.280] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:13:57.318+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:13:57.318] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:13:57.319+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:13:57.319] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 44969 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:13:58.164+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:58.164] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:58.294+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:13:58.294] clusterConfig unchanged [2021-10-22T02:13:58.501+0000] [.error] [src/action/start.go:func1:93] [103] [02:13:58.501] Error starting mongod : [02:13:58.501] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44969,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 44969 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:13:57.349Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 66108 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:13:57.369+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:13:57.369+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:13:57.369+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44969,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:13:57.371+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:13:57.373+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:13:57.376+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:13:57.401+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:13:57.401+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:13:57.402+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:13:57.447+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:13:57.447+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:13:57.983+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868837:983828][66108:0x7f3ce028ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:13:58.094+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868838:94840][66108:0x7f3ce028ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:13:58.195+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868838:195837][66108:0x7f3ce028ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T02:13:58.314+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868838:314341][66108:0x7f3ce028ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:13:58.394+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868838:394067][66108:0x7f3ce028ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:13:58.453+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868838:453174][66108:0x7f3ce028ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:13:58.453+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868838:453230][66108:0x7f3ce028ebc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:13:58.501+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:13:58.501] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:44969 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:13:58.501] Error starting mongod : [02:13:58.501] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44969,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 44969 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:13:57.349Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 66108 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:13:57.369+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:13:57.369+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:13:57.369+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44969,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:13:57.371+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:13:57.373+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:13:57.376+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:13:57.401+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:13:57.401+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:13:57.402+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:13:57.447+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:13:57.447+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:13:57.983+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868837:983828][66108:0x7f3ce028ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:13:58.094+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:13:58.502+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:13:58.501] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:13:58.501] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:44969 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:13:58.501] Error starting mongod : [02:13:58.501] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44969,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 44969 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:13:57.349Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 66108 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:13:57.369+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:13:57.369+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:13:57.369+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44969,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:13:57.371+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:13:57.373+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:13:57.376+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:13:57.401+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:13:57.401+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:13:57.402+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:13:57.447+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:13:57.447+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:13:57.983+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868837:983828][66108:0x7f3ce028ebc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:13:58.502+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:13:58.502] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:13:58.502+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:13:58.502] Initiating stopEphemeralMongo [2021-10-22T02:13:58.502+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:13:58.502] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:13:58.502+0000] [.error] [src/director/director.go:executePlan:962] [02:13:58.502] Failed to apply action. Result = : [02:13:58.501] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:13:58.501] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:44969 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:13:58.501] Error starting mongod : [02:13:58.501] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44969,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 44969 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:13:57.349Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 66108 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:13:57.369+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:13:57.369+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:13:57.369+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44969,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:13:57.371+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:13:57.373+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:13:57.376+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:13:57.401+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:13:57.401+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:13:57.402+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:13:57.447+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:13:57.447+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:13:57.983+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:13:58.502+0000] [.error] [src/director/director.go:planAndExecute:567] [02:13:58.502] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:13:58.502] Failed to apply action. Result = : [02:13:58.501] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:13:58.501] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:44969 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:13:58.501] Error starting mongod : [02:13:58.501] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44969,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 44969 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:13:57.349Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 66108 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:13:57.369+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:13:57.369+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:13:57.369+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44969,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:13:57.371+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:13:57.373+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:13:57.376+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:13:57.401+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:13:57.401+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:13:57.402+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:13:57.447+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:13:57.447+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:13:58.502+0000] [.error] [src/director/director.go:mainLoop:399] [02:13:58.502] Failed to planAndExecute : [02:13:58.502] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:13:58.502] Failed to apply action. Result = : [02:13:58.501] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:13:58.501] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:44969 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:13:58.501] Error starting mongod : [02:13:58.501] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44969,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 44969 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:13:57.349Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 66108 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:13:57.369+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:13:57.369+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:13:57.369+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44969,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:13:57.371+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:13:57.373+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:13:57.376+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:13:57.401+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:13:57.401+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:13:57.402+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:13:57.447+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:13:57.447+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:13:59.164+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:13:59.164] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:13:59.534+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:13:59.534] clusterConfig unchanged [2021-10-22T02:14:00.165+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:00.165] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:01.165+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:01.165] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:02.163+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:02.163] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:03.166+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:03.166] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:04.166+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:04.166] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:05.167+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:05.167] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:06.168+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:06.167] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:07.169+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:07.169] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:07.566+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:14:07.566] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.78562564s [2021-10-22T02:14:08.167+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:08.167] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:09.167+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:09.167] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:09.636+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:14:09.636] clusterConfig unchanged [2021-10-22T02:14:10.167+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:10.167] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:11.168+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:11.168] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:12.168+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:12.168] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:13.169+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:13.169] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:14.169+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:14.169] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:15.169+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:15.169] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:16.169+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:16.169] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:17.170+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:17.170] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:18.171+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:18.171] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:19.171+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:19.171] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:19.788+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:14:19.788] clusterConfig unchanged [2021-10-22T02:14:20.172+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:20.172] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:21.173+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:21.173] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:22.173+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:22.173] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:23.180+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:23.180] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:24.173+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:24.173] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:25.173+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:25.173] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:26.174+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:26.174] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:27.174+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:27.174] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:28.175+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:28.175] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:29.175+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:29.175] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:29.906+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:14:29.906] clusterConfig unchanged [2021-10-22T02:14:30.176+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:30.176] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:31.176+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:31.176] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:32.177+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:32.177] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:33.177+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:33.177] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:34.178+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:34.177] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:35.185+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:35.185] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:36.179+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:36.179] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:37.179+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:37.179] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:38.179+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:38.179] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:39.179+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:39.179] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:40.008+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:14:40.008] clusterConfig unchanged [2021-10-22T02:14:40.180+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:40.180] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:41.181+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:41.181] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:41.389+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:14:41.389] memory:rss=57135104 alloc=12195696 totalalloc=9786201208 mallocs=124824072 frees=124723271 heapinuse=17506304 heapobjects=100801 gcpausetotalns=1604946501 gcpauselastns=126840 gcnum=1453 utimens=207363488000 stimens=80634163000 minflt=522421 majflt=405 [2021-10-22T02:14:42.182+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:42.182] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:43.182+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:43.182] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:44.186+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:44.186] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:44.399+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [02:14:44.399] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44969 (local=false) because it is not up [2021-10-22T02:14:45.182+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:45.182] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:46.183+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:46.183] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:47.184+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:47.184] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:48.185+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:48.185] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:49.185+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:49.185] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:50.120+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:14:50.120] clusterConfig unchanged [2021-10-22T02:14:50.186+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:50.186] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:51.187+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:51.187] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:52.187+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:52.187] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:53.188+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:53.188] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:54.188+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:54.188] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:55.195+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:55.195] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:56.190+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:56.190] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:57.190+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:57.190] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:58.193+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:58.193] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:58.686+0000] [.info] [src/director/director.go:computePlan:280] [02:14:58.686] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:14:58.686+0000] [.info] [src/director/director.go:tracef:794] [02:14:58.686] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:14:58.686+0000] [.info] [src/director/director.go:tracef:794] [02:14:58.686] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:14:58.687+0000] [.info] [src/director/director.go:tracef:794] [02:14:58.687] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:14:58.687+0000] [.info] [src/director/director.go:tracef:794] [02:14:58.687] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:14:58.687+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:14:58.687] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:14:58.860+0000] [.info] [src/director/director.go:tracef:794] [02:14:58.860] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:14:58.860+0000] [.info] [src/director/director.go:tracef:794] [02:14:58.860] because [2021-10-22T02:14:58.860+0000] [.info] [src/director/director.go:tracef:794] [02:14:58.860] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:14:58.860+0000] [.info] [src/director/director.go:tracef:794] [02:14:58.860] because ['currentState.CanBounceStop' = true] [2021-10-22T02:14:59.031+0000] [.info] [src/director/director.go:tracef:794] [02:14:59.031] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:14:59.031+0000] [.info] [src/director/director.go:tracef:794] [02:14:59.031] because [2021-10-22T02:14:59.031+0000] [.info] [src/director/director.go:tracef:794] [02:14:59.031] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:14:59.031+0000] [.info] [src/director/director.go:tracef:794] [02:14:59.031] because ['currentState.Up' = false] [2021-10-22T02:14:59.031+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:14:59.031] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:14:59.031+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:14:59.031] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:14:59.119+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:14:59.119] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:14:59.127+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:14:59.127] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:14:59.128+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:14:59.128] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:14:59.129+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:14:59.129] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:14:59.130+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:14:59.130] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:14:59.130+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:14:59.130] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:14:59.130+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:14:59.130] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:14:59.131+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:14:59.131] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:14:59.194+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:14:59.194] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:14:59.371+0000] [.info] [src/director/director.go:tracef:794] [02:14:59.371] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:14:59.371+0000] [.info] [src/director/director.go:tracef:794] [02:14:59.371] because ['currentState.Up' = false] [2021-10-22T02:14:59.560+0000] [.info] [src/director/director.go:tracef:794] [02:14:59.560] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:14:59.561+0000] [.info] [src/director/director.go:tracef:794] [02:14:59.561] because ['currentState.Up' = false] [2021-10-22T02:14:59.561+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:14:59.561] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:14:59.783+0000] [.info] [src/director/director.go:tracef:794] [02:14:59.783] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:14:59.783+0000] [.info] [src/director/director.go:tracef:794] [02:14:59.783] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:15:00.000+0000] [.info] [src/director/director.go:tracef:794] [02:15:00.000] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:15:00.000+0000] [.info] [src/director/director.go:tracef:794] [02:15:00.000] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:15:00.000+0000] [.info] [src/director/director.go:tracef:794] [02:15:00.000] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:15:00.000+0000] [.info] [src/director/director.go:tracef:794] [02:15:00.000] because ['currentState.Up' = false] [2021-10-22T02:15:00.000+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:15:00.000] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:15:00.000+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:15:00.000] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:15:00.040+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:15:00.040] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:15:00.040+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:15:00.040] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40863 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:15:00.196+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:00.196] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:00.321+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:15:00.321] clusterConfig unchanged [2021-10-22T02:15:00.802+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:15:00.802] ...Finished start [pid=11160] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40863,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:15:00.935+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:15:00.934] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 132.297876ms. [2021-10-22T02:15:00.935+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:15:00.935] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40863 (local=false) [2021-10-22T02:15:00.935+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:15:00.935] Dropping existing documents from system.replset [2021-10-22T02:15:01.066+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:15:01.066] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:15:01.071+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:15:01.071] Inserting oplogTruncateAfterPoint document [2021-10-22T02:15:01.071+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:15:01.071] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:15:01.072+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:15:01.072] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:15:01.072+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:15:01.072] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:15:01.085+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:15:01.085] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:15:01.085+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:15:01.085] Setting the minvalid document to [{_id ObjectID("61721ea5523370002becc7e0")} {t -1} {ts {0 1}}] [2021-10-22T02:15:01.097+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:15:01.097] Inserted [{_id ObjectID("61721ea5523370002becc7e0")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:15:01.098+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:15:01.097] Inserting new replica set configuration document. [2021-10-22T02:15:01.098+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:15:01.098] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:15:01.111+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:15:01.111] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:15:01.114+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:15:01.114] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:15:01.114+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:15:01.114] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:15:01.114+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:15:01.114] Initiating stopEphemeralMongo [2021-10-22T02:15:01.114+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:15:01.114] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40863 (local=false) [2021-10-22T02:15:01.114+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:15:01.114] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40863 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:15:01.201+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:01.201] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:01.443+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:15:01.443] clusterConfig unchanged [2021-10-22T02:15:01.524+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:15:01.524] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.649714ms. [2021-10-22T02:15:01.524+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:15:01.524] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40863 (local=false) [2021-10-22T02:15:01.524+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:15:01.524] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.28µs. [2021-10-22T02:15:01.525+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:15:01.524] Successfully finished running stopEphemeralMongo [2021-10-22T02:15:01.525+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:15:01.525] Successfully stopped existing ephemeral mongod. [2021-10-22T02:15:01.525+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:15:01.525] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:15:01.525+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:15:01.525] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:15:01.561+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:15:01.560] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:15:01.561+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:15:01.561] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48559 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:15:02.203+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:02.203] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:02.543+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:15:02.543] clusterConfig unchanged [2021-10-22T02:15:02.892+0000] [.error] [src/action/start.go:func1:93] [103] [02:15:02.892] Error starting mongod : [02:15:02.892] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48559,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48559 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 12585 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:15:01.666+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:15:01.666+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:15:01.666+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48559,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:15:01.668+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:15:01.669+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:15:01.670+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:15:01.682+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:15:01.682+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:15:01.703+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:15:01.748+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:15:01.748+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:15:02.308+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868902:308910][12585:0x7fc4fdf53bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:15:02.417+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868902:417198][12585:0x7fc4fdf53bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:15:02.542+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868902:542110][12585:0x7fc4fdf53bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:15:02.695+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868902:695083][12585:0x7fc4fdf53bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:15:02.772+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868902:772193][12585:0x7fc4fdf53bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:15:02.841+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868902:841388][12585:0x7fc4fdf53bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:15:02.841+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868902:841465][12585:0x7fc4fdf53bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:15:02.843+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:15:02.892+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:15:02.892] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48559 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:15:02.892] Error starting mongod : [02:15:02.892] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48559,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48559 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 12585 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:15:01.666+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:15:01.666+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:15:01.666+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48559,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:15:01.668+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:15:01.669+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:15:01.670+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:15:01.682+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:15:01.682+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:15:01.703+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:15:01.748+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:15:01.748+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:15:02.308+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868902:308910][12585:0x7fc4fdf53bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:15:02.417+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868902:417198][12585:0x7fc4fdf53bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:15:02.893+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:15:02.893] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:15:02.892] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48559 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:15:02.892] Error starting mongod : [02:15:02.892] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48559,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48559 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 12585 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:15:01.666+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:15:01.666+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:15:01.666+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48559,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:15:01.668+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:15:01.669+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:15:01.670+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:15:01.682+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:15:01.682+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:15:01.703+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:15:01.748+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:15:01.748+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:15:02.308+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868902:308910][12585:0x7fc4fdf53bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:15:02.417+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:15:02.893+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:15:02.893] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:15:02.893+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:15:02.893] Initiating stopEphemeralMongo [2021-10-22T02:15:02.893+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:15:02.893] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:15:02.893+0000] [.error] [src/director/director.go:executePlan:962] [02:15:02.893] Failed to apply action. Result = : [02:15:02.893] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:15:02.892] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48559 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:15:02.892] Error starting mongod : [02:15:02.892] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48559,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48559 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 12585 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:15:01.666+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:15:01.666+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:15:01.666+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48559,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:15:01.668+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:15:01.669+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:15:01.670+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:15:01.682+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:15:01.682+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:15:01.703+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:15:01.748+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:15:01.748+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:15:02.308+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868902:308910][12585:0x7fc4fdf53bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:15:02.893+0000] [.error] [src/director/director.go:planAndExecute:567] [02:15:02.893] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:15:02.893] Failed to apply action. Result = : [02:15:02.893] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:15:02.892] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48559 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:15:02.892] Error starting mongod : [02:15:02.892] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48559,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48559 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 12585 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:15:01.666+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:15:01.666+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:15:01.666+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48559,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:15:01.668+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:15:01.669+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:15:01.670+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:15:01.682+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:15:01.682+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:15:01.703+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:15:01.748+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:15:01.748+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:15:02.308+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:15:02.893+0000] [.error] [src/director/director.go:mainLoop:399] [02:15:02.893] Failed to planAndExecute : [02:15:02.893] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:15:02.893] Failed to apply action. Result = : [02:15:02.893] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:15:02.892] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48559 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:15:02.892] Error starting mongod : [02:15:02.892] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48559,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48559 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 12585 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:15:01.666+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:15:01.666+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:15:01.666+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48559,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:15:01.668+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:15:01.669+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:15:01.670+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:15:01.682+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:15:01.682+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:15:01.703+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:15:01.748+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:15:01.748+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:15:02.308+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:15:03.202+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:03.202] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:03.777+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:15:03.776] clusterConfig unchanged [2021-10-22T02:15:04.211+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:04.211] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:05.212+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:05.212] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:06.221+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:06.221] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:07.221+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:07.221] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:08.222+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:08.222] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:09.222+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:09.222] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:10.222+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:10.222] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:11.224+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:11.224] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:12.146+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:15:12.146] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 13.015513832s [2021-10-22T02:15:12.221+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:12.221] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:13.222+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:13.222] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:13.957+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:15:13.957] clusterConfig unchanged [2021-10-22T02:15:14.230+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:14.230] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:15.223+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:15.223] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:16.223+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:16.223] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:17.224+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:17.224] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:18.224+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:18.224] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:19.225+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:19.225] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:20.225+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:20.225] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:21.225+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:21.225] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:22.225+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:22.225] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:23.226+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:23.226] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:24.060+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:15:24.060] clusterConfig unchanged [2021-10-22T02:15:24.227+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:24.227] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:25.227+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:25.227] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:26.232+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:26.232] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:27.228+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:27.228] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:28.228+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:28.228] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:29.230+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:29.230] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:30.230+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:30.230] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:31.234+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:31.234] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:32.231+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:32.231] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:33.231+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:33.231] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:34.161+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:15:34.161] clusterConfig unchanged [2021-10-22T02:15:34.233+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:34.233] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:35.233+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:35.233] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:36.234+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:36.234] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:37.234+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:37.234] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:38.243+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:38.243] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:39.236+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:39.236] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:40.236+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:40.236] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:41.237+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:41.237] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:41.389+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:15:41.389] memory:rss=55803904 alloc=10422304 totalalloc=9877560456 mallocs=126001003 frees=125920379 heapinuse=16908288 heapobjects=80624 gcpausetotalns=1608987042 gcpauselastns=82104 gcnum=1465 utimens=209825375000 stimens=81819837000 minflt=526904 majflt=405 [2021-10-22T02:15:42.238+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:42.238] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:43.238+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:43.238] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:44.239+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:44.239] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:44.274+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:15:44.273] clusterConfig unchanged [2021-10-22T02:15:45.239+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:45.239] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:46.239+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:46.239] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:47.246+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:47.246] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:48.240+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:48.240] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:49.240+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:49.240] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:50.240+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:50.240] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:51.241+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:51.241] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:52.241+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:52.241] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:53.241+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:53.241] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:54.242+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:54.242] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:54.366+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:15:54.366] clusterConfig unchanged [2021-10-22T02:15:55.243+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:55.243] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:56.247+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:56.247] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:57.245+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:57.245] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:58.245+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:58.245] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:15:59.253+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:15:59.253] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:00.245+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:00.245] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:01.246+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:01.246] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:02.247+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:02.247] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:03.075+0000] [.info] [src/director/director.go:computePlan:280] [02:16:03.075] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:16:03.076+0000] [.info] [src/director/director.go:tracef:794] [02:16:03.076] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:16:03.076+0000] [.info] [src/director/director.go:tracef:794] [02:16:03.076] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:16:03.076+0000] [.info] [src/director/director.go:tracef:794] [02:16:03.076] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:16:03.076+0000] [.info] [src/director/director.go:tracef:794] [02:16:03.076] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:16:03.076+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:16:03.076] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:16:03.234+0000] [.info] [src/director/director.go:tracef:794] [02:16:03.234] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:16:03.234+0000] [.info] [src/director/director.go:tracef:794] [02:16:03.234] because [2021-10-22T02:16:03.234+0000] [.info] [src/director/director.go:tracef:794] [02:16:03.234] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:16:03.234+0000] [.info] [src/director/director.go:tracef:794] [02:16:03.234] because ['currentState.CanBounceStop' = true] [2021-10-22T02:16:03.253+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:03.253] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:03.388+0000] [.info] [src/director/director.go:tracef:794] [02:16:03.388] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:16:03.388+0000] [.info] [src/director/director.go:tracef:794] [02:16:03.388] because [2021-10-22T02:16:03.388+0000] [.info] [src/director/director.go:tracef:794] [02:16:03.388] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:16:03.388+0000] [.info] [src/director/director.go:tracef:794] [02:16:03.388] because ['currentState.Up' = false] [2021-10-22T02:16:03.388+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:16:03.388] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:16:03.388+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:16:03.388] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:16:03.474+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:16:03.474] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:16:03.474+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:16:03.474] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:16:03.482+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:16:03.482] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:16:03.483+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:16:03.483] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:16:03.484+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:16:03.484] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:16:03.484+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:16:03.484] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:16:03.484+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:16:03.484] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:16:03.485+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:16:03.485] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:16:03.724+0000] [.info] [src/director/director.go:tracef:794] [02:16:03.724] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:16:03.724+0000] [.info] [src/director/director.go:tracef:794] [02:16:03.724] because ['currentState.Up' = false] [2021-10-22T02:16:03.959+0000] [.info] [src/director/director.go:tracef:794] [02:16:03.959] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:16:03.959+0000] [.info] [src/director/director.go:tracef:794] [02:16:03.959] because ['currentState.Up' = false] [2021-10-22T02:16:03.959+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:16:03.959] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:16:04.194+0000] [.info] [src/director/director.go:tracef:794] [02:16:04.194] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:16:04.194+0000] [.info] [src/director/director.go:tracef:794] [02:16:04.194] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:16:04.250+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:04.250] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:04.436+0000] [.info] [src/director/director.go:tracef:794] [02:16:04.436] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:16:04.436+0000] [.info] [src/director/director.go:tracef:794] [02:16:04.436] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:16:04.437+0000] [.info] [src/director/director.go:tracef:794] [02:16:04.437] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:16:04.437+0000] [.info] [src/director/director.go:tracef:794] [02:16:04.437] because ['currentState.Up' = false] [2021-10-22T02:16:04.437+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:16:04.437] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:16:04.437+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:16:04.437] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:16:04.469+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:16:04.469] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:16:04.469+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:16:04.469] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56353 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:16:04.646+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:16:04.646] clusterConfig unchanged [2021-10-22T02:16:05.222+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:16:05.222] ...Finished start [pid=32035] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56353,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:16:05.251+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:05.251] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:05.334+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:16:05.334] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 111.401704ms. [2021-10-22T02:16:05.334+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:16:05.334] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:56353 (local=false) [2021-10-22T02:16:05.334+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:16:05.334] Dropping existing documents from system.replset [2021-10-22T02:16:05.485+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:16:05.485] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:16:05.492+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:16:05.492] Inserting oplogTruncateAfterPoint document [2021-10-22T02:16:05.492+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:16:05.492] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:16:05.493+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:16:05.493] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:16:05.493+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:16:05.493] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:16:05.506+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:16:05.506] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:16:05.506+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:16:05.506] Setting the minvalid document to [{_id ObjectID("61721ee5523370002becc7ef")} {t -1} {ts {0 1}}] [2021-10-22T02:16:05.518+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:16:05.518] Inserted [{_id ObjectID("61721ee5523370002becc7ef")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:16:05.518+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:16:05.518] Inserting new replica set configuration document. [2021-10-22T02:16:05.518+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:16:05.518] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:16:05.530+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:16:05.530] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:16:05.534+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:16:05.534] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:16:05.534+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:16:05.534] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:16:05.534+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:16:05.534] Initiating stopEphemeralMongo [2021-10-22T02:16:05.534+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:16:05.534] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:56353 (local=false) [2021-10-22T02:16:05.535+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:16:05.534] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:56353 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:16:05.768+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:16:05.768] clusterConfig unchanged [2021-10-22T02:16:05.932+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:16:05.932] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.169389ms. [2021-10-22T02:16:05.932+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:16:05.932] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:56353 (local=false) [2021-10-22T02:16:05.932+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:16:05.932] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.385µs. [2021-10-22T02:16:05.933+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:16:05.933] Successfully finished running stopEphemeralMongo [2021-10-22T02:16:05.933+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:16:05.933] Successfully stopped existing ephemeral mongod. [2021-10-22T02:16:05.933+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:16:05.933] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:16:05.933+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:16:05.933] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:16:05.966+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:16:05.966] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:16:05.967+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:16:05.967] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51699 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:16:06.254+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:06.254] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:06.881+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:16:06.881] clusterConfig unchanged [2021-10-22T02:16:07.223+0000] [.error] [src/action/start.go:func1:93] [103] [02:16:07.223] Error starting mongod : [02:16:07.222] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51699,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51699 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:16:05.997Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 33608 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:16:06.016+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:16:06.016+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:16:06.016+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51699,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:16:06.018+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:16:06.019+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:16:06.020+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:16:06.030+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:16:06.031+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:16:06.031+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:16:06.086+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:16:06.086+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:16:06.694+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868966:694766][33608:0x7efd24d67bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:16:06.796+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868966:796874][33608:0x7efd24d67bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:16:06.899+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868966:899552][33608:0x7efd24d67bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T02:16:07.012+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868967:12621][33608:0x7efd24d67bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:16:07.086+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868967:86172][33608:0x7efd24d67bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:16:07.152+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868967:152191][33608:0x7efd24d67bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:16:07.152+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868967:152266][33608:0x7efd24d67bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:16:07.223+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:16:07.223] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51699 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:16:07.223] Error starting mongod : [02:16:07.222] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51699,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51699 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:16:05.997Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 33608 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:16:06.016+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:16:06.016+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:16:06.016+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51699,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:16:06.018+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:16:06.019+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:16:06.020+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:16:06.030+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:16:06.031+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:16:06.031+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:16:06.086+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:16:06.086+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:16:06.694+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868966:694766][33608:0x7efd24d67bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:16:06.796+00:00"},"s":"I...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:16:07.223+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:16:07.223] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:16:07.223] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51699 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:16:07.223] Error starting mongod : [02:16:07.222] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51699,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51699 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:16:05.997Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 33608 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:16:06.016+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:16:06.016+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:16:06.016+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51699,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:16:06.018+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:16:06.019+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:16:06.020+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:16:06.030+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:16:06.031+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:16:06.031+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:16:06.086+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:16:06.086+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:16:06.694+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634868966:694766][33608:0x7efd24d67bc0], t... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:16:07.223+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:16:07.223] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:16:07.223+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:16:07.223] Initiating stopEphemeralMongo [2021-10-22T02:16:07.223+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:16:07.223] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:16:07.224+0000] [.error] [src/director/director.go:executePlan:962] [02:16:07.224] Failed to apply action. Result = : [02:16:07.223] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:16:07.223] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51699 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:16:07.223] Error starting mongod : [02:16:07.222] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51699,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51699 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:16:05.997Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 33608 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:16:06.016+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:16:06.016+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:16:06.016+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51699,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:16:06.018+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:16:06.019+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:16:06.020+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:16:06.030+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:16:06.031+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:16:06.031+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:16:06.086+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:16:06.086+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:16:06.694+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","ms... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:16:07.224+0000] [.error] [src/director/director.go:planAndExecute:567] [02:16:07.224] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:16:07.224] Failed to apply action. Result = : [02:16:07.223] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:16:07.223] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51699 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:16:07.223] Error starting mongod : [02:16:07.222] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51699,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51699 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:16:05.997Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 33608 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:16:06.016+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:16:06.016+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:16:06.016+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51699,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:16:06.018+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:16:06.019+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:16:06.020+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:16:06.030+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:16:06.031+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:16:06.031+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:16:06.086+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:16:06.086+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"c... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:16:07.224+0000] [.error] [src/director/director.go:mainLoop:399] [02:16:07.224] Failed to planAndExecute : [02:16:07.224] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:16:07.224] Failed to apply action. Result = : [02:16:07.223] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:16:07.223] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51699 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:16:07.223] Error starting mongod : [02:16:07.222] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51699,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51699 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:16:05.997Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 33608 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:16:06.016+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:16:06.016+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:16:06.016+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51699,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:16:06.018+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:16:06.019+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:16:06.020+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:16:06.030+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:16:06.031+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:16:06.031+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:16:06.086+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:16:06.086+00:00"},"s":"I", "c":"NETWORK", "id":5771... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:16:07.254+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:07.254] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:08.151+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:16:08.151] clusterConfig unchanged [2021-10-22T02:16:08.252+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:08.252] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:09.254+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:09.254] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:10.255+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:10.255] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:11.256+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:11.256] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:12.255+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:12.255] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:13.256+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:13.256] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:14.255+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:14.255] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:15.257+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:15.257] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:16.149+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:16:16.149] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.663694658s [2021-10-22T02:16:16.254+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:16.254] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:17.255+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:17.255] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:18.255+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:18.255] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:18.371+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:16:18.371] clusterConfig unchanged [2021-10-22T02:16:18.565+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:16:18.565] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:16:18.574+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:18.574] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:16:18.574+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [02:16:18.574] Could not get version : [02:16:18.574] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:16:18.574+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [02:16:18.574] Error getting version for log lines. Skipping : [02:16:18.574] Could not get version : [02:16:18.574] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:16:19.256+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:19.256] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:20.257+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:20.257] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:21.258+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:21.258] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:22.258+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:22.258] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:23.259+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:23.259] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:24.259+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:24.259] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:25.260+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:25.260] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:26.261+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:26.261] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:27.262+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:27.262] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:28.263+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:28.263] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:28.493+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:16:28.493] clusterConfig unchanged [2021-10-22T02:16:29.264+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:29.264] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:30.264+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:30.264] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:31.265+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:31.265] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:32.265+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:32.265] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:33.269+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:33.269] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:34.269+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:34.269] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:35.267+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:35.267] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:36.267+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:36.267] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:37.268+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:37.268] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:38.268+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:38.268] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:38.624+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:16:38.624] clusterConfig unchanged [2021-10-22T02:16:39.269+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:39.269] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:40.270+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:40.270] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:41.269+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:41.269] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:41.390+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:16:41.390] memory:rss=56819712 alloc=14790728 totalalloc=9969924160 mallocs=127173669 frees=127043593 heapinuse=18530304 heapobjects=130076 gcpausetotalns=1611685906 gcpauselastns=104328 gcnum=1476 utimens=212144706000 stimens=82835780000 minflt=532765 majflt=405 [2021-10-22T02:16:42.278+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:42.278] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:43.271+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:43.271] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:44.272+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:44.272] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:45.272+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:45.272] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:46.272+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:46.272] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:47.273+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:47.273] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:48.274+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:48.274] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:48.762+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:16:48.762] clusterConfig unchanged [2021-10-22T02:16:49.274+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:49.274] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:50.275+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:50.275] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:51.275+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:51.275] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:52.276+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:52.276] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:53.277+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:53.277] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:54.278+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:54.278] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:55.278+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:55.278] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:56.279+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:56.279] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:57.279+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:57.279] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:58.280+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:58.280] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:16:58.917+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:16:58.917] clusterConfig unchanged [2021-10-22T02:16:59.281+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:16:59.281] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:00.282+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:00.282] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:01.285+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:01.285] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:02.291+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:02.291] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:03.284+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:03.284] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:04.284+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:04.284] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:05.285+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:05.285] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:06.285+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:06.285] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:07.300+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:07.300] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:07.416+0000] [.info] [src/director/director.go:computePlan:280] [02:17:07.416] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:17:07.416+0000] [.info] [src/director/director.go:tracef:794] [02:17:07.416] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:17:07.416+0000] [.info] [src/director/director.go:tracef:794] [02:17:07.416] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:17:07.416+0000] [.info] [src/director/director.go:tracef:794] [02:17:07.416] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:17:07.416+0000] [.info] [src/director/director.go:tracef:794] [02:17:07.416] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:17:07.416+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:17:07.416] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:17:07.576+0000] [.info] [src/director/director.go:tracef:794] [02:17:07.576] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:17:07.576+0000] [.info] [src/director/director.go:tracef:794] [02:17:07.576] because [2021-10-22T02:17:07.576+0000] [.info] [src/director/director.go:tracef:794] [02:17:07.576] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:17:07.576+0000] [.info] [src/director/director.go:tracef:794] [02:17:07.576] because ['currentState.CanBounceStop' = true] [2021-10-22T02:17:07.729+0000] [.info] [src/director/director.go:tracef:794] [02:17:07.729] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:17:07.729+0000] [.info] [src/director/director.go:tracef:794] [02:17:07.729] because [2021-10-22T02:17:07.729+0000] [.info] [src/director/director.go:tracef:794] [02:17:07.729] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:17:07.730+0000] [.info] [src/director/director.go:tracef:794] [02:17:07.729] because ['currentState.Up' = false] [2021-10-22T02:17:07.730+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:17:07.730] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:17:07.730+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:17:07.730] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:17:07.809+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:17:07.809] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:17:07.817+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:17:07.817] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:17:07.818+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:17:07.818] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:17:07.819+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:17:07.819] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:17:07.819+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:17:07.819] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:17:07.819+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:17:07.819] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:17:07.820+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:17:07.820] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:17:08.052+0000] [.info] [src/director/director.go:tracef:794] [02:17:08.052] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:17:08.052+0000] [.info] [src/director/director.go:tracef:794] [02:17:08.052] because ['currentState.Up' = false] [2021-10-22T02:17:08.283+0000] [.info] [src/director/director.go:tracef:794] [02:17:08.283] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:17:08.283+0000] [.info] [src/director/director.go:tracef:794] [02:17:08.283] because ['currentState.Up' = false] [2021-10-22T02:17:08.283+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:17:08.283] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:17:08.291+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:08.291] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:08.476+0000] [.info] [src/director/director.go:tracef:794] [02:17:08.476] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:17:08.476+0000] [.info] [src/director/director.go:tracef:794] [02:17:08.476] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:17:08.683+0000] [.info] [src/director/director.go:tracef:794] [02:17:08.683] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:17:08.683+0000] [.info] [src/director/director.go:tracef:794] [02:17:08.683] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:17:08.683+0000] [.info] [src/director/director.go:tracef:794] [02:17:08.683] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:17:08.683+0000] [.info] [src/director/director.go:tracef:794] [02:17:08.683] because ['currentState.Up' = false] [2021-10-22T02:17:08.683+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:17:08.683] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:17:08.683+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:17:08.683] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:17:08.720+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:17:08.720] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:17:08.720+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:17:08.720] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60600 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:17:09.099+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:17:09.099] clusterConfig unchanged [2021-10-22T02:17:09.294+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:09.294] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:09.444+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:17:09.444] ...Finished start [pid=52651] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60600,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:17:09.574+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:17:09.574] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 129.321282ms. [2021-10-22T02:17:09.574+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:17:09.574] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:60600 (local=false) [2021-10-22T02:17:09.574+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:17:09.574] Dropping existing documents from system.replset [2021-10-22T02:17:09.717+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:17:09.717] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:17:09.730+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:17:09.730] Inserting oplogTruncateAfterPoint document [2021-10-22T02:17:09.730+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:17:09.730] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:17:09.732+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:17:09.732] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:17:09.732+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:17:09.732] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:17:09.745+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:17:09.745] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:17:09.745+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:17:09.745] Setting the minvalid document to [{_id ObjectID("61721f25523370002becc7fd")} {t -1} {ts {0 1}}] [2021-10-22T02:17:09.757+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:17:09.757] Inserted [{_id ObjectID("61721f25523370002becc7fd")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:17:09.757+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:17:09.757] Inserting new replica set configuration document. [2021-10-22T02:17:09.757+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:17:09.757] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:17:09.770+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:17:09.770] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:17:09.773+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:17:09.773] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:17:09.773+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:17:09.773] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:17:09.773+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:17:09.773] Initiating stopEphemeralMongo [2021-10-22T02:17:09.773+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:17:09.773] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:60600 (local=false) [2021-10-22T02:17:09.773+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:17:09.773] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:60600 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:17:10.211+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:17:10.211] clusterConfig unchanged [2021-10-22T02:17:10.250+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:17:10.250] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.589683ms. [2021-10-22T02:17:10.250+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:17:10.250] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:60600 (local=false) [2021-10-22T02:17:10.250+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:17:10.250] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.903µs. [2021-10-22T02:17:10.250+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:17:10.250] Successfully finished running stopEphemeralMongo [2021-10-22T02:17:10.250+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:17:10.250] Successfully stopped existing ephemeral mongod. [2021-10-22T02:17:10.250+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:17:10.250] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:17:10.250+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:17:10.250] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:17:10.295+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:17:10.295] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:17:10.296+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:17:10.296] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37450 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:17:10.298+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:10.298] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:11.298+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:11.298] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:11.359+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:17:11.359] clusterConfig unchanged [2021-10-22T02:17:11.484+0000] [.error] [src/action/start.go:func1:93] [103] [02:17:11.484] Error starting mongod : [02:17:11.484] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37450 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:17:10.325Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 54212 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:17:10.344+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:17:10.344+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:17:10.344+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:17:10.346+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:17:10.347+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:17:10.348+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:17:10.366+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:17:10.366+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:17:10.377+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:17:10.422+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:17:10.422+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:17:10.960+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869030:960833][54212:0x7ff32b468bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:17:11.067+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869031:67027][54212:0x7ff32b468bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:17:11.173+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869031:173468][54212:0x7ff32b468bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:17:11.287+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869031:287946][54212:0x7ff32b468bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:17:11.366+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869031:366294][54212:0x7ff32b468bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:17:11.432+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869031:432010][54212:0x7ff32b468bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:17:11.432+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869031:432083][54212:0x7ff32b468bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:17:11.485+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:17:11.484] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37450 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:17:11.484] Error starting mongod : [02:17:11.484] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37450 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:17:10.325Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 54212 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:17:10.344+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:17:10.344+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:17:10.344+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:17:10.346+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:17:10.347+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:17:10.348+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:17:10.366+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:17:10.366+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:17:10.377+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:17:10.422+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:17:10.422+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:17:10.960+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869030:960833][54212:0x7ff32b468bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:17:11.067+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:17:11.485+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:17:11.485] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:17:11.484] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37450 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:17:11.484] Error starting mongod : [02:17:11.484] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37450 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:17:10.325Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 54212 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:17:10.344+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:17:10.344+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:17:10.344+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:17:10.346+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:17:10.347+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:17:10.348+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:17:10.366+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:17:10.366+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:17:10.377+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:17:10.422+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:17:10.422+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:17:10.960+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869030:960833][54212:0x7ff32b468bc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:17:11.485+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:17:11.485] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:17:11.485+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:17:11.485] Initiating stopEphemeralMongo [2021-10-22T02:17:11.485+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:17:11.485] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:17:11.486+0000] [.error] [src/director/director.go:executePlan:962] [02:17:11.486] Failed to apply action. Result = : [02:17:11.485] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:17:11.484] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37450 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:17:11.484] Error starting mongod : [02:17:11.484] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37450 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:17:10.325Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 54212 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:17:10.344+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:17:10.344+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:17:10.344+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:17:10.346+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:17:10.347+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:17:10.348+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:17:10.366+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:17:10.366+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:17:10.377+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:17:10.422+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:17:10.422+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:17:10.960+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:17:11.486+0000] [.error] [src/director/director.go:planAndExecute:567] [02:17:11.486] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:17:11.486] Failed to apply action. Result = : [02:17:11.485] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:17:11.484] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37450 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:17:11.484] Error starting mongod : [02:17:11.484] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37450 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:17:10.325Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 54212 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:17:10.344+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:17:10.344+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:17:10.344+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:17:10.346+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:17:10.347+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:17:10.348+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:17:10.366+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:17:10.366+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:17:10.377+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:17:10.422+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:17:10.422+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:17:11.486+0000] [.error] [src/director/director.go:mainLoop:399] [02:17:11.486] Failed to planAndExecute : [02:17:11.486] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:17:11.486] Failed to apply action. Result = : [02:17:11.485] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:17:11.484] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37450 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:17:11.484] Error starting mongod : [02:17:11.484] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37450 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:17:10.325Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 54212 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:17:10.344+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:17:10.344+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:17:10.344+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37450,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:17:10.346+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:17:10.347+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:17:10.348+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:17:10.366+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:17:10.366+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:17:10.377+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:17:10.422+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:17:10.422+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:17:12.298+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:12.298] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:12.584+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:17:12.584] clusterConfig unchanged [2021-10-22T02:17:13.298+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:13.298] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:14.300+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:14.300] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:15.300+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:15.300] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:16.299+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:16.299] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:17.302+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:17.302] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:18.300+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:18.300] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:19.302+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:19.302] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:20.302+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:20.302] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:20.604+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:17:20.604] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.783905699s [2021-10-22T02:17:21.300+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:21.300] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:22.300+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:22.300] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:22.705+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:17:22.705] clusterConfig unchanged [2021-10-22T02:17:23.301+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:23.301] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:24.302+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:24.302] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:25.302+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:25.302] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:26.303+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:26.303] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:27.303+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:27.303] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:27.570+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:17:27.570] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:17:27.571+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:17:27.571] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:17:28.304+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:28.304] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:29.305+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:29.305] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:30.305+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:30.305] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:31.306+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:31.306] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:32.306+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:32.306] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:32.916+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:17:32.916] clusterConfig unchanged [2021-10-22T02:17:33.307+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:33.307] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:34.308+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:34.308] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:35.308+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:35.308] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:36.309+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:36.309] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:37.310+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:37.310] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:38.312+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [02:17:38.312] lsblk cache invalidated, running lsblk command [2021-10-22T02:17:38.314+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:38.314] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:39.310+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:39.310] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:40.311+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:40.311] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:41.312+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:41.312] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:41.392+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:17:41.392] memory:rss=56741888 alloc=14953544 totalalloc=10062191464 mallocs=128358728 frees=128227617 heapinuse=18579456 heapobjects=131111 gcpausetotalns=1615761223 gcpauselastns=60079 gcnum=1488 utimens=214448039000 stimens=83855340000 minflt=538080 majflt=405 [2021-10-22T02:17:42.317+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:42.317] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:43.029+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:17:43.029] clusterConfig unchanged [2021-10-22T02:17:43.317+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:43.317] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:44.318+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:44.318] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:44.407+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [02:17:44.407] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37450 (local=false) because it is not up [2021-10-22T02:17:45.318+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:45.318] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:46.319+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:46.319] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:47.319+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:47.319] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:48.320+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:48.320] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:49.321+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:49.320] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:50.321+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:50.321] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:51.322+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:51.322] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:52.330+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:52.330] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:53.161+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:17:53.161] clusterConfig unchanged [2021-10-22T02:17:53.322+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:53.322] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:54.323+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:54.323] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:55.324+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:55.324] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:56.325+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:56.325] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:57.325+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:57.325] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:58.327+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:58.327] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:17:59.326+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:17:59.326] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:00.327+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:00.327] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:01.328+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:01.328] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:02.329+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:02.329] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:03.282+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:18:03.282] clusterConfig unchanged [2021-10-22T02:18:03.335+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:03.335] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:04.330+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:04.330] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:05.330+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:05.330] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:06.330+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:06.330] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:07.331+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:07.331] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:08.332+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:08.332] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:09.333+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:09.333] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:10.333+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:10.333] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:11.333+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:11.333] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:11.670+0000] [.info] [src/director/director.go:computePlan:280] [02:18:11.670] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:18:11.670+0000] [.info] [src/director/director.go:tracef:794] [02:18:11.670] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:18:11.670+0000] [.info] [src/director/director.go:tracef:794] [02:18:11.670] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:18:11.670+0000] [.info] [src/director/director.go:tracef:794] [02:18:11.670] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:18:11.670+0000] [.info] [src/director/director.go:tracef:794] [02:18:11.670] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:18:11.671+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:18:11.671] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:18:11.844+0000] [.info] [src/director/director.go:tracef:794] [02:18:11.844] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:18:11.844+0000] [.info] [src/director/director.go:tracef:794] [02:18:11.844] because [2021-10-22T02:18:11.844+0000] [.info] [src/director/director.go:tracef:794] [02:18:11.844] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:18:11.844+0000] [.info] [src/director/director.go:tracef:794] [02:18:11.844] because ['currentState.CanBounceStop' = true] [2021-10-22T02:18:11.996+0000] [.info] [src/director/director.go:tracef:794] [02:18:11.996] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:18:11.996+0000] [.info] [src/director/director.go:tracef:794] [02:18:11.996] because [2021-10-22T02:18:11.996+0000] [.info] [src/director/director.go:tracef:794] [02:18:11.996] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:18:11.996+0000] [.info] [src/director/director.go:tracef:794] [02:18:11.996] because ['currentState.Up' = false] [2021-10-22T02:18:11.996+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:18:11.996] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:18:11.996+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:18:11.996] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:18:12.085+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:18:12.085] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:18:12.093+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:18:12.093] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:18:12.094+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:18:12.094] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:18:12.094+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:18:12.094] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:18:12.096+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:18:12.096] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:18:12.096+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:18:12.096] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:18:12.096+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:18:12.096] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:18:12.098+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:18:12.098] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:18:12.316+0000] [.info] [src/director/director.go:tracef:794] [02:18:12.316] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:18:12.316+0000] [.info] [src/director/director.go:tracef:794] [02:18:12.316] because ['currentState.Up' = false] [2021-10-22T02:18:12.364+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:12.364] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:12.539+0000] [.info] [src/director/director.go:tracef:794] [02:18:12.539] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:18:12.539+0000] [.info] [src/director/director.go:tracef:794] [02:18:12.539] because ['currentState.Up' = false] [2021-10-22T02:18:12.539+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:18:12.539] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:18:12.748+0000] [.info] [src/director/director.go:tracef:794] [02:18:12.747] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:18:12.748+0000] [.info] [src/director/director.go:tracef:794] [02:18:12.748] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:18:12.968+0000] [.info] [src/director/director.go:tracef:794] [02:18:12.968] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:18:12.969+0000] [.info] [src/director/director.go:tracef:794] [02:18:12.968] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:18:12.969+0000] [.info] [src/director/director.go:tracef:794] [02:18:12.969] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:18:12.969+0000] [.info] [src/director/director.go:tracef:794] [02:18:12.969] because ['currentState.Up' = false] [2021-10-22T02:18:12.969+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:18:12.969] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:18:12.969+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:18:12.969] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:18:13.006+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:18:13.006] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:18:13.006+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:18:13.006] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37325 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:18:13.345+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:13.345] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:13.497+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:18:13.497] clusterConfig unchanged [2021-10-22T02:18:13.727+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:18:13.727] ...Finished start [pid=73449] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37325,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:18:13.905+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:18:13.905] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 177.661954ms. [2021-10-22T02:18:13.905+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:18:13.905] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37325 (local=false) [2021-10-22T02:18:13.905+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:18:13.905] Dropping existing documents from system.replset [2021-10-22T02:18:14.087+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:18:14.087] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:18:14.094+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:18:14.094] Inserting oplogTruncateAfterPoint document [2021-10-22T02:18:14.094+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:18:14.094] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:18:14.096+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:18:14.096] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:18:14.096+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:18:14.096] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:18:14.108+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:18:14.108] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:18:14.108+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:18:14.108] Setting the minvalid document to [{_id ObjectID("61721f66523370002becc80b")} {t -1} {ts {0 1}}] [2021-10-22T02:18:14.142+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:18:14.142] Inserted [{_id ObjectID("61721f66523370002becc80b")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:18:14.142+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:18:14.142] Inserting new replica set configuration document. [2021-10-22T02:18:14.142+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:18:14.142] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:18:14.154+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:18:14.154] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:18:14.197+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:18:14.197] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:18:14.197+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:18:14.197] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:18:14.197+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:18:14.197] Initiating stopEphemeralMongo [2021-10-22T02:18:14.197+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:18:14.197] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37325 (local=false) [2021-10-22T02:18:14.197+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:18:14.197] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37325 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:18:14.357+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:14.357] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:14.636+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:18:14.636] clusterConfig unchanged [2021-10-22T02:18:14.654+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:18:14.654] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.558ms. [2021-10-22T02:18:14.654+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:18:14.654] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37325 (local=false) [2021-10-22T02:18:14.655+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:18:14.655] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 96.884µs. [2021-10-22T02:18:14.655+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:18:14.655] Successfully finished running stopEphemeralMongo [2021-10-22T02:18:14.655+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:18:14.655] Successfully stopped existing ephemeral mongod. [2021-10-22T02:18:14.655+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:18:14.655] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:18:14.655+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:18:14.655] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:18:14.699+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:18:14.699] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:18:14.699+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:18:14.699] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51504 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:18:15.347+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:15.347] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:15.758+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:18:15.758] clusterConfig unchanged [2021-10-22T02:18:15.941+0000] [.error] [src/action/start.go:func1:93] [103] [02:18:15.941] Error starting mongod : [02:18:15.940] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51504,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51504 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 989 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: h1p-config-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:18:14.748+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:18:14.748+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:18:14.748+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51504,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:18:14.750+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:18:14.750+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:18:14.752+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:18:14.776+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:18:14.776+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:18:14.784+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:18:14.829+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:18:14.829+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:18:15.390+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869095:390454][989:0x7f265717abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:18:15.519+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869095:519210][989:0x7f265717abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:18:15.618+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869095:618628][989:0x7f265717abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T02:18:15.733+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869095:733351][989:0x7f265717abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:18:15.811+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869095:811559][989:0x7f265717abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:18:15.883+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869095:883009][989:0x7f265717abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:18:15.883+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869095:883081][989:0x7f265717abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:18:15.895+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:18:15.941+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:18:15.941] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51504 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:18:15.941] Error starting mongod : [02:18:15.940] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51504,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51504 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 989 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: h1p-config-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:18:14.748+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:18:14.748+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:18:14.748+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51504,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:18:14.750+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:18:14.750+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:18:14.752+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:18:14.776+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:18:14.776+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:18:14.784+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:18:14.829+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:18:14.829+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:18:15.390+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869095:390454][989:0x7f265717abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:18:15.519+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869095:519210][989:0x7f265717abc0], txn-re...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:18:15.941+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:18:15.941] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:18:15.941] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51504 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:18:15.941] Error starting mongod : [02:18:15.940] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51504,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51504 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 989 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: h1p-config-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:18:14.748+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:18:14.748+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:18:14.748+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51504,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:18:14.750+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:18:14.750+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:18:14.752+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:18:14.776+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:18:14.776+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:18:14.784+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:18:14.829+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:18:14.829+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:18:15.390+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869095:390454][989:0x7f265717abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:18:15.519+00:00"},"s":"I", "c":"STORAGE", "id":2... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:18:15.941+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:18:15.941] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:18:15.941+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:18:15.941] Initiating stopEphemeralMongo [2021-10-22T02:18:15.941+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:18:15.941] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:18:15.942+0000] [.error] [src/director/director.go:executePlan:962] [02:18:15.942] Failed to apply action. Result = : [02:18:15.941] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:18:15.941] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51504 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:18:15.941] Error starting mongod : [02:18:15.940] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51504,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51504 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 989 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: h1p-config-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:18:14.748+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:18:14.748+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:18:14.748+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51504,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:18:14.750+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:18:14.750+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:18:14.752+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:18:14.776+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:18:14.776+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:18:14.784+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:18:14.829+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:18:14.829+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:18:15.390+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869095:390454][989:0x7f265717abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 throu... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:18:15.942+0000] [.error] [src/director/director.go:planAndExecute:567] [02:18:15.942] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:18:15.942] Failed to apply action. Result = : [02:18:15.941] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:18:15.941] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51504 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:18:15.941] Error starting mongod : [02:18:15.940] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51504,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51504 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 989 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: h1p-config-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:18:14.748+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:18:14.748+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:18:14.748+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51504,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:18:14.750+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:18:14.750+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:18:14.752+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:18:14.776+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:18:14.776+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:18:14.784+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:18:14.829+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:18:14.829+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:18:15.390+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"Wired... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:18:15.942+0000] [.error] [src/director/director.go:mainLoop:399] [02:18:15.942] Failed to planAndExecute : [02:18:15.942] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:18:15.942] Failed to apply action. Result = : [02:18:15.941] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:18:15.941] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51504 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:18:15.941] Error starting mongod : [02:18:15.940] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51504,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51504 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 989 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: h1p-config-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:18:14.748+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:18:14.748+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:18:14.748+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51504,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:18:14.750+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:18:14.750+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:18:14.752+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:18:14.776+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:18:14.776+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:18:14.784+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:18:14.829+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:18:14.829+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:18:15.390+00:00"},"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:18:16.348+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:16.348] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:16.998+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:18:16.998] clusterConfig unchanged [2021-10-22T02:18:17.348+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:17.348] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:18.348+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:18.348] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:19.350+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:19.350] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:20.350+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:20.350] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:21.358+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:21.358] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:22.350+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:22.350] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:23.352+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:23.352] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:24.352+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:24.352] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:24.909+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:18:24.909] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.810680101s [2021-10-22T02:18:25.352+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:25.351] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:26.361+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:26.361] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:27.140+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:18:27.140] clusterConfig unchanged [2021-10-22T02:18:27.362+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:27.362] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:28.359+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:28.359] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:29.362+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:29.362] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:30.360+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:30.360] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:31.361+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:31.361] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:32.368+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:32.368] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:33.364+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:33.364] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:34.370+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:34.370] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:35.370+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:35.370] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:36.369+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:36.368] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:37.271+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:18:37.271] clusterConfig unchanged [2021-10-22T02:18:37.370+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:37.370] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:38.370+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:38.370] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:39.372+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:39.372] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:40.371+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:40.371] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:41.374+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:41.374] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:41.393+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:18:41.393] memory:rss=57159680 alloc=13886440 totalalloc=10153351552 mallocs=129535870 frees=129415990 heapinuse=18235392 heapobjects=119880 gcpausetotalns=1617859604 gcpauselastns=96137 gcnum=1500 utimens=217361846000 stimens=85201476000 minflt=543333 majflt=405 [2021-10-22T02:18:42.374+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:42.374] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:43.375+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:43.375] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:44.376+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:44.376] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:45.375+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:45.375] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:46.376+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:46.375] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:47.376+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:47.376] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:47.638+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:18:47.638] clusterConfig unchanged [2021-10-22T02:18:48.377+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:48.377] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:49.378+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:49.378] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:50.379+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:50.378] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:51.380+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:51.380] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:52.380+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:52.380] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:53.381+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:53.381] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:54.381+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:54.381] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:55.382+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:55.382] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:56.385+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:56.385] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:57.383+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:57.383] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:57.741+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:18:57.741] clusterConfig unchanged [2021-10-22T02:18:58.384+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:58.384] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:18:59.385+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:18:59.385] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:00.385+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:00.385] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:01.390+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:01.390] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:02.387+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:02.387] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:03.386+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:03.386] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:04.394+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:04.394] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:05.387+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:05.387] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:06.388+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:06.388] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:07.389+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:07.389] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:07.872+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:19:07.872] clusterConfig unchanged [2021-10-22T02:19:08.389+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:08.389] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:09.390+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:09.390] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:10.390+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:10.390] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:11.390+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:11.390] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:12.391+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:12.391] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:13.391+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:13.391] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:14.392+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:14.392] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:15.393+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:15.393] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:16.110+0000] [.info] [src/director/director.go:computePlan:280] [02:19:16.110] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:19:16.110+0000] [.info] [src/director/director.go:tracef:794] [02:19:16.110] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:19:16.110+0000] [.info] [src/director/director.go:tracef:794] [02:19:16.110] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:19:16.110+0000] [.info] [src/director/director.go:tracef:794] [02:19:16.110] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:19:16.110+0000] [.info] [src/director/director.go:tracef:794] [02:19:16.110] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:19:16.110+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:19:16.110] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:19:16.272+0000] [.info] [src/director/director.go:tracef:794] [02:19:16.272] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:19:16.272+0000] [.info] [src/director/director.go:tracef:794] [02:19:16.272] because [2021-10-22T02:19:16.272+0000] [.info] [src/director/director.go:tracef:794] [02:19:16.272] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:19:16.272+0000] [.info] [src/director/director.go:tracef:794] [02:19:16.272] because ['currentState.CanBounceStop' = true] [2021-10-22T02:19:16.396+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:16.396] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:16.428+0000] [.info] [src/director/director.go:tracef:794] [02:19:16.428] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:19:16.428+0000] [.info] [src/director/director.go:tracef:794] [02:19:16.428] because [2021-10-22T02:19:16.428+0000] [.info] [src/director/director.go:tracef:794] [02:19:16.428] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:19:16.428+0000] [.info] [src/director/director.go:tracef:794] [02:19:16.428] because ['currentState.Up' = false] [2021-10-22T02:19:16.428+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:19:16.428] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:19:16.428+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:19:16.428] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:19:16.513+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:19:16.513] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:19:16.521+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:19:16.521] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:19:16.522+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:19:16.522] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:19:16.522+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:19:16.522] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:19:16.523+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:19:16.523] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:19:16.524+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:19:16.524] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:19:16.524+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:19:16.524] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:19:16.526+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:19:16.526] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:19:16.752+0000] [.info] [src/director/director.go:tracef:794] [02:19:16.752] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:19:16.752+0000] [.info] [src/director/director.go:tracef:794] [02:19:16.752] because ['currentState.Up' = false] [2021-10-22T02:19:16.973+0000] [.info] [src/director/director.go:tracef:794] [02:19:16.973] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:19:16.973+0000] [.info] [src/director/director.go:tracef:794] [02:19:16.973] because ['currentState.Up' = false] [2021-10-22T02:19:16.973+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:19:16.973] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:19:17.192+0000] [.info] [src/director/director.go:tracef:794] [02:19:17.192] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:19:17.192+0000] [.info] [src/director/director.go:tracef:794] [02:19:17.192] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:19:17.397+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:17.397] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:17.397+0000] [.info] [src/director/director.go:tracef:794] [02:19:17.397] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:19:17.398+0000] [.info] [src/director/director.go:tracef:794] [02:19:17.397] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:19:17.398+0000] [.info] [src/director/director.go:tracef:794] [02:19:17.398] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:19:17.398+0000] [.info] [src/director/director.go:tracef:794] [02:19:17.398] because ['currentState.Up' = false] [2021-10-22T02:19:17.398+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:19:17.398] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:19:17.398+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:19:17.398] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:19:17.432+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:19:17.432] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:19:17.433+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:19:17.433] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 40554 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:19:18.073+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:19:18.073] clusterConfig unchanged [2021-10-22T02:19:18.156+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:19:18.156] ...Finished start [pid=20738] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":40554,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:19:18.337+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:19:18.337] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 180.732802ms. [2021-10-22T02:19:18.337+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:19:18.337] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40554 (local=false) [2021-10-22T02:19:18.337+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:19:18.337] Dropping existing documents from system.replset [2021-10-22T02:19:18.400+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:18.400] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:18.475+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:19:18.475] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:19:18.478+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:19:18.478] Inserting oplogTruncateAfterPoint document [2021-10-22T02:19:18.478+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:19:18.478] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:19:18.480+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:19:18.480] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:19:18.480+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:19:18.480] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:19:18.491+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:19:18.491] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:19:18.491+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:19:18.491] Setting the minvalid document to [{_id ObjectID("61721fa6523370002becc819")} {t -1} {ts {0 1}}] [2021-10-22T02:19:18.504+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:19:18.504] Inserted [{_id ObjectID("61721fa6523370002becc819")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:19:18.504+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:19:18.504] Inserting new replica set configuration document. [2021-10-22T02:19:18.504+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:19:18.504] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:19:18.516+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:19:18.516] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:19:18.530+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:19:18.530] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:19:18.530+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:19:18.530] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:19:18.530+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:19:18.530] Initiating stopEphemeralMongo [2021-10-22T02:19:18.530+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:19:18.530] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40554 (local=false) [2021-10-22T02:19:18.530+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:19:18.530] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40554 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:19:18.911+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:19:18.911] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.095606ms. [2021-10-22T02:19:18.911+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:19:18.911] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:40554 (local=false) [2021-10-22T02:19:18.911+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:19:18.911] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.329µs. [2021-10-22T02:19:18.911+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:19:18.911] Successfully finished running stopEphemeralMongo [2021-10-22T02:19:18.911+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:19:18.911] Successfully stopped existing ephemeral mongod. [2021-10-22T02:19:18.911+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:19:18.911] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:19:18.911+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:19:18.911] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:19:18.945+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:19:18.945] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:19:18.945+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:19:18.945] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38688 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:19:19.186+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:19:19.186] clusterConfig unchanged [2021-10-22T02:19:19.406+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:19.406] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:20.143+0000] [.error] [src/action/start.go:func1:93] [103] [02:19:20.143] Error starting mongod : [02:19:20.143] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38688,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38688 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:19:18.977Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 22138 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:19:19.002+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:19:19.002+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:19:19.002+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38688,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:19:19.003+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:19:19.004+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:19:19.005+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:19:19.067+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:19:19.067+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:19:19.067+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:19:19.113+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:19:19.113+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:19:19.624+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869159:624940][22138:0x7f4ece082bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:19:19.726+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869159:726812][22138:0x7f4ece082bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:19:19.829+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869159:829840][22138:0x7f4ece082bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T02:19:19.945+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869159:945671][22138:0x7f4ece082bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:19:20.023+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869160:23939][22138:0x7f4ece082bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:19:20.090+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869160:90939][22138:0x7f4ece082bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:19:20.091+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869160:91026][22138:0x7f4ece082bc0], txn-recover: [WT_VERB_RECOVERY | WT_...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:19:20.143+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:19:20.143] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38688 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:19:20.143] Error starting mongod : [02:19:20.143] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38688,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38688 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:19:18.977Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 22138 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:19:19.002+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:19:19.002+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:19:19.002+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38688,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:19:19.003+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:19:19.004+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:19:19.005+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:19:19.067+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:19:19.067+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:19:19.067+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:19:19.113+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:19:19.113+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:19:19.624+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869159:624940][22138:0x7f4ece082bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:19:19.726+00:00"},"s":...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:19:20.144+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:19:20.144] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:19:20.143] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38688 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:19:20.143] Error starting mongod : [02:19:20.143] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38688,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38688 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:19:18.977Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 22138 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:19:19.002+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:19:19.002+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:19:19.002+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38688,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:19:19.003+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:19:19.004+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:19:19.005+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:19:19.067+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:19:19.067+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:19:19.067+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:19:19.113+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:19:19.113+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:19:19.624+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869159:624940][22138:0x7f4ece082bc0],... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:19:20.144+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:19:20.144] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:19:20.144+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:19:20.144] Initiating stopEphemeralMongo [2021-10-22T02:19:20.144+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:19:20.144] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:19:20.145+0000] [.error] [src/director/director.go:executePlan:962] [02:19:20.145] Failed to apply action. Result = : [02:19:20.144] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:19:20.143] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38688 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:19:20.143] Error starting mongod : [02:19:20.143] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38688,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38688 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:19:18.977Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 22138 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:19:19.002+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:19:19.002+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:19:19.002+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38688,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:19:19.003+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:19:19.004+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:19:19.005+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:19:19.067+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:19:19.067+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:19:19.067+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:19:19.113+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:19:19.113+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:19:19.624+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:19:20.145+0000] [.error] [src/director/director.go:planAndExecute:567] [02:19:20.145] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:19:20.145] Failed to apply action. Result = : [02:19:20.144] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:19:20.143] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38688 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:19:20.143] Error starting mongod : [02:19:20.143] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38688,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38688 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:19:18.977Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 22138 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:19:19.002+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:19:19.002+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:19:19.002+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38688,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:19:19.003+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:19:19.004+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:19:19.005+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:19:19.067+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:19:19.067+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:19:19.067+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:19:19.113+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:19:19.113+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:19:20.145+0000] [.error] [src/director/director.go:mainLoop:399] [02:19:20.145] Failed to planAndExecute : [02:19:20.145] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:19:20.145] Failed to apply action. Result = : [02:19:20.144] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:19:20.143] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38688 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:19:20.143] Error starting mongod : [02:19:20.143] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38688,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38688 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:19:18.977Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 22138 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 0-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:19:19.002+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:19:19.002+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:19:19.002+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38688,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:19:19.003+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:19:19.004+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:19:19.005+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:19:19.067+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:19:19.067+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:19:19.067+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:19:19.113+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:19:19.113+00:00"},"s":"I", "c":"NETWORK", "id":57... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:19:20.405+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:20.405] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:20.449+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:19:20.449] clusterConfig unchanged [2021-10-22T02:19:21.405+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:21.405] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:22.406+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:22.406] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:23.404+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:23.404] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:24.407+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:24.407] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:25.408+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:25.408] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:26.408+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:26.408] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:27.417+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:27.417] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:28.420+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:28.420] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:29.337+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:19:29.337] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.811664369s [2021-10-22T02:19:29.418+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:29.418] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:30.417+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:30.417] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:30.577+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:19:30.577] clusterConfig unchanged [2021-10-22T02:19:31.420+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:31.420] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:32.419+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:32.419] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:33.427+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:33.427] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:34.420+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:34.420] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:35.420+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:35.420] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:36.422+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:36.422] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:37.422+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:37.422] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:38.424+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:38.424] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:39.423+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:39.423] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:40.424+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:40.424] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:40.699+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:19:40.699] clusterConfig unchanged [2021-10-22T02:19:41.393+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:19:41.393] memory:rss=58089472 alloc=13130104 totalalloc=10245317752 mallocs=130716962 frees=130605917 heapinuse=18145280 heapobjects=111045 gcpausetotalns=1620967030 gcpauselastns=53982 gcnum=1512 utimens=219944548000 stimens=86382620000 minflt=548206 majflt=405 [2021-10-22T02:19:41.424+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:41.424] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:42.425+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:42.425] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:43.426+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:43.426] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:44.429+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:44.429] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:45.427+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:45.427] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:46.429+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:46.429] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:47.429+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:47.429] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:48.434+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:48.434] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:49.430+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:49.430] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:50.430+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:50.430] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:50.852+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:19:50.852] clusterConfig unchanged [2021-10-22T02:19:51.430+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:51.430] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:52.430+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:52.430] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:53.430+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:53.430] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:54.431+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:54.431] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:55.432+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:55.432] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:56.432+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:56.432] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:57.432+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:57.432] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:58.432+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:58.432] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:19:59.433+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:19:59.433] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:00.434+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:00.434] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:00.964+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:20:00.964] clusterConfig unchanged [2021-10-22T02:20:01.434+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:01.434] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:02.454+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:02.454] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:03.436+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:03.436] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:04.436+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:04.436] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:05.444+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:05.444] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:06.437+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:06.437] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:07.438+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:07.438] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:08.438+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:08.438] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:09.440+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:09.440] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:10.439+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:10.439] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:11.066+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:20:11.066] clusterConfig unchanged [2021-10-22T02:20:11.440+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:11.439] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:12.440+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:12.440] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:13.440+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:13.440] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:14.441+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:14.441] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:15.441+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:15.441] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:16.444+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:16.444] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:17.442+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:17.442] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:18.443+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:18.443] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:19.445+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:19.445] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:20.366+0000] [.info] [src/director/director.go:computePlan:280] [02:20:20.366] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:20:20.367+0000] [.info] [src/director/director.go:tracef:794] [02:20:20.367] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:20:20.368+0000] [.info] [src/director/director.go:tracef:794] [02:20:20.368] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:20:20.368+0000] [.info] [src/director/director.go:tracef:794] [02:20:20.368] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:20:20.368+0000] [.info] [src/director/director.go:tracef:794] [02:20:20.368] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:20:20.368+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:20:20.368] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:20:20.468+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:20.468] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:20.600+0000] [.info] [src/director/director.go:tracef:794] [02:20:20.600] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:20:20.600+0000] [.info] [src/director/director.go:tracef:794] [02:20:20.600] because [2021-10-22T02:20:20.600+0000] [.info] [src/director/director.go:tracef:794] [02:20:20.600] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:20:20.600+0000] [.info] [src/director/director.go:tracef:794] [02:20:20.600] because ['currentState.CanBounceStop' = true] [2021-10-22T02:20:20.760+0000] [.info] [src/director/director.go:tracef:794] [02:20:20.760] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:20:20.760+0000] [.info] [src/director/director.go:tracef:794] [02:20:20.760] because [2021-10-22T02:20:20.760+0000] [.info] [src/director/director.go:tracef:794] [02:20:20.760] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:20:20.760+0000] [.info] [src/director/director.go:tracef:794] [02:20:20.760] because ['currentState.Up' = false] [2021-10-22T02:20:20.761+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:20:20.761] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:20:20.761+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:20:20.761] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:20:20.952+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:20:20.952] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:20:20.960+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:20:20.960] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:20:20.961+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:20:20.961] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:20:20.962+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:20:20.962] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:20:20.962+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:20:20.962] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:20:20.962+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:20:20.962] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:20:20.964+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:20:20.964] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:20:21.189+0000] [.info] [src/director/director.go:tracef:794] [02:20:21.189] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:20:21.189+0000] [.info] [src/director/director.go:tracef:794] [02:20:21.189] because ['currentState.Up' = false] [2021-10-22T02:20:21.253+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:20:21.253] clusterConfig unchanged [2021-10-22T02:20:21.405+0000] [.info] [src/director/director.go:tracef:794] [02:20:21.405] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:20:21.405+0000] [.info] [src/director/director.go:tracef:794] [02:20:21.405] because ['currentState.Up' = false] [2021-10-22T02:20:21.405+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:20:21.405] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:20:21.450+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:21.450] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:21.637+0000] [.info] [src/director/director.go:tracef:794] [02:20:21.637] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:20:21.637+0000] [.info] [src/director/director.go:tracef:794] [02:20:21.637] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:20:21.853+0000] [.info] [src/director/director.go:tracef:794] [02:20:21.853] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:20:21.853+0000] [.info] [src/director/director.go:tracef:794] [02:20:21.853] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:20:21.854+0000] [.info] [src/director/director.go:tracef:794] [02:20:21.854] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:20:21.854+0000] [.info] [src/director/director.go:tracef:794] [02:20:21.854] because ['currentState.Up' = false] [2021-10-22T02:20:21.854+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:20:21.854] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:20:21.854+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:20:21.854] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:20:21.895+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:20:21.895] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:20:21.895+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:20:21.895] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37345 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:20:22.390+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:20:22.390] clusterConfig unchanged [2021-10-22T02:20:22.451+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:22.450] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:22.631+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:20:22.631] ...Finished start [pid=41457] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37345,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:20:22.785+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:20:22.785] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 153.001857ms. [2021-10-22T02:20:22.785+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:20:22.785] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37345 (local=false) [2021-10-22T02:20:22.785+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:20:22.785] Dropping existing documents from system.replset [2021-10-22T02:20:22.930+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:20:22.930] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:20:22.933+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:20:22.933] Inserting oplogTruncateAfterPoint document [2021-10-22T02:20:22.933+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:20:22.933] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:20:22.934+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:20:22.934] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:20:22.934+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:20:22.934] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:20:22.947+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:20:22.947] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:20:22.948+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:20:22.947] Setting the minvalid document to [{_id ObjectID("61721fe6523370002becc827")} {t -1} {ts {0 1}}] [2021-10-22T02:20:22.960+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:20:22.960] Inserted [{_id ObjectID("61721fe6523370002becc827")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:20:22.960+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:20:22.960] Inserting new replica set configuration document. [2021-10-22T02:20:22.960+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:20:22.960] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:20:22.974+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:20:22.974] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:20:22.978+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:20:22.978] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:20:22.978+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:20:22.978] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:20:22.978+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:20:22.978] Initiating stopEphemeralMongo [2021-10-22T02:20:22.978+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:20:22.978] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37345 (local=false) [2021-10-22T02:20:22.978+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:20:22.978] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37345 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:20:23.406+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:20:23.406] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.169906ms. [2021-10-22T02:20:23.406+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:20:23.406] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37345 (local=false) [2021-10-22T02:20:23.407+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:20:23.406] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 50.117µs. [2021-10-22T02:20:23.407+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:20:23.407] Successfully finished running stopEphemeralMongo [2021-10-22T02:20:23.407+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:20:23.407] Successfully stopped existing ephemeral mongod. [2021-10-22T02:20:23.407+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:20:23.407] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:20:23.407+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:20:23.407] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:20:23.444+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:20:23.444] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:20:23.444+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:20:23.444] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59640 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:20:23.452+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:23.452] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:23.512+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:20:23.512] clusterConfig unchanged [2021-10-22T02:20:24.452+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:24.452] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:24.644+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:20:24.644] clusterConfig unchanged [2021-10-22T02:20:24.706+0000] [.error] [src/action/start.go:func1:93] [103] [02:20:24.706] Error starting mongod : [02:20:24.706] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59640,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59640 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 43051 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:20:23.494+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:20:23.494+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:20:23.494+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59640,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:20:23.496+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:20:23.497+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:20:23.498+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:20:23.514+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:20:23.514+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:20:23.514+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:20:23.570+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:20:23.570+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:20:24.136+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869224:136752][43051:0x7fc0c0e9cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:20:24.261+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869224:261665][43051:0x7fc0c0e9cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:20:24.369+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869224:368992][43051:0x7fc0c0e9cbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:20:24.491+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869224:491862][43051:0x7fc0c0e9cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:20:24.579+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869224:579654][43051:0x7fc0c0e9cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:20:24.640+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869224:640657][43051:0x7fc0c0e9cbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:20:24.640+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869224:640728][43051:0x7fc0c0e9cbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:20:24.645+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:20:24.706+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:20:24.706] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59640 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:20:24.706] Error starting mongod : [02:20:24.706] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59640,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59640 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 43051 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:20:23.494+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:20:23.494+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:20:23.494+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59640,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:20:23.496+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:20:23.497+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:20:23.498+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:20:23.514+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:20:23.514+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:20:23.514+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:20:23.570+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:20:23.570+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:20:24.136+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869224:136752][43051:0x7fc0c0e9cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:20:24.261+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869224:261665][43051:0x7fc0c0e9cbc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:20:24.707+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:20:24.707] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:20:24.706] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59640 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:20:24.706] Error starting mongod : [02:20:24.706] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59640,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59640 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 43051 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:20:23.494+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:20:23.494+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:20:23.494+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59640,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:20:23.496+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:20:23.497+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:20:23.498+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:20:23.514+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:20:23.514+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:20:23.514+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:20:23.570+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:20:23.570+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:20:24.136+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869224:136752][43051:0x7fc0c0e9cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:20:24.261+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:20:24.707+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:20:24.707] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:20:24.707+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:20:24.707] Initiating stopEphemeralMongo [2021-10-22T02:20:24.707+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:20:24.707] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:20:24.707+0000] [.error] [src/director/director.go:executePlan:962] [02:20:24.707] Failed to apply action. Result = : [02:20:24.707] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:20:24.706] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59640 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:20:24.706] Error starting mongod : [02:20:24.706] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59640,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59640 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 43051 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:20:23.494+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:20:23.494+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:20:23.494+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59640,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:20:23.496+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:20:23.497+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:20:23.498+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:20:23.514+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:20:23.514+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:20:23.514+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:20:23.570+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:20:23.570+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:20:24.136+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869224:136752][43051:0x7fc0c0e9cbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:20:24.708+0000] [.error] [src/director/director.go:planAndExecute:567] [02:20:24.708] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:20:24.707] Failed to apply action. Result = : [02:20:24.707] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:20:24.706] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59640 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:20:24.706] Error starting mongod : [02:20:24.706] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59640,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59640 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 43051 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:20:23.494+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:20:23.494+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:20:23.494+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59640,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:20:23.496+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:20:23.497+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:20:23.498+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:20:23.514+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:20:23.514+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:20:23.514+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:20:23.570+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:20:23.570+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:20:24.136+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:20:24.708+0000] [.error] [src/director/director.go:mainLoop:399] [02:20:24.708] Failed to planAndExecute : [02:20:24.708] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:20:24.707] Failed to apply action. Result = : [02:20:24.707] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:20:24.706] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59640 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:20:24.706] Error starting mongod : [02:20:24.706] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59640,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59640 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 43051 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:20:23.494+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:20:23.494+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:20:23.494+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59640,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:20:23.496+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:20:23.497+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:20:23.498+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:20:23.514+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:20:23.514+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:20:23.514+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:20:23.570+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:20:23.570+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:20:24.136+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:20:25.451+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:25.451] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:25.894+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:20:25.894] clusterConfig unchanged [2021-10-22T02:20:26.450+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:26.450] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:27.451+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:27.451] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:28.452+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:28.452] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:29.453+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:29.453] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:30.452+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:30.452] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:31.459+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:31.459] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:32.454+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:32.454] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:33.453+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:33.453] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:33.694+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:20:33.694] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.729990851s [2021-10-22T02:20:34.453+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:34.452] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:35.453+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:35.453] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:36.036+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:20:36.036] clusterConfig unchanged [2021-10-22T02:20:36.454+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:36.454] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:37.457+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:37.457] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:38.455+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:38.455] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:39.455+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:39.455] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:40.455+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:40.455] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:41.394+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:20:41.394] memory:rss=58032128 alloc=14909384 totalalloc=10337708416 mallocs=131906667 frees=131776247 heapinuse=18849792 heapobjects=130420 gcpausetotalns=1628396601 gcpauselastns=99968 gcnum=1524 utimens=222321597000 stimens=87456773000 minflt=553490 majflt=405 [2021-10-22T02:20:41.455+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:41.455] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:42.456+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:42.456] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:43.457+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:43.457] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:44.415+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [02:20:44.415] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:59640 (local=false) because it is not up [2021-10-22T02:20:44.457+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:44.457] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:45.457+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:45.457] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:46.157+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:20:46.157] clusterConfig unchanged [2021-10-22T02:20:46.458+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:46.458] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:47.458+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:47.458] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:48.459+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:48.459] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:49.459+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:49.459] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:50.460+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:50.460] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:51.461+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:51.461] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:52.463+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:52.463] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:53.467+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:53.467] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:54.462+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:54.462] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:55.464+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:55.464] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:56.298+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:20:56.298] clusterConfig unchanged [2021-10-22T02:20:56.463+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:56.463] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:57.464+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:57.464] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:58.464+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:58.464] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:20:59.464+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:20:59.464] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:00.464+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:00.464] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:01.465+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:01.465] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:02.466+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:02.466] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:03.467+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:03.467] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:04.467+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:04.467] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:05.476+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:05.476] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:06.422+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:21:06.422] clusterConfig unchanged [2021-10-22T02:21:06.468+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:06.468] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:07.469+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:07.469] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:08.470+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:08.470] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:09.471+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:09.471] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:10.471+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:10.471] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:11.472+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:11.472] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:12.473+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:12.473] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:13.474+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:13.474] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:14.477+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:14.477] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:15.475+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:15.475] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:16.482+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:16.482] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:16.611+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:21:16.611] clusterConfig unchanged [2021-10-22T02:21:17.475+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:17.475] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:18.476+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:18.476] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:18.565+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:21:18.565] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:21:18.567+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:18.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:21:18.567+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [02:21:18.567] Could not get version : [02:21:18.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:21:18.567+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [02:21:18.567] Error getting version for log lines. Skipping : [02:21:18.567] Could not get version : [02:21:18.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:21:19.477+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:19.477] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:20.478+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:20.478] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:21.478+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:21.478] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:22.479+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:22.479] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:23.479+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:23.479] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:24.480+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:24.480] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:24.893+0000] [.info] [src/director/director.go:computePlan:280] [02:21:24.893] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:21:24.893+0000] [.info] [src/director/director.go:tracef:794] [02:21:24.893] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:21:24.893+0000] [.info] [src/director/director.go:tracef:794] [02:21:24.893] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:21:24.894+0000] [.info] [src/director/director.go:tracef:794] [02:21:24.894] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:21:24.894+0000] [.info] [src/director/director.go:tracef:794] [02:21:24.894] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:21:24.894+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:21:24.894] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:21:25.100+0000] [.info] [src/director/director.go:tracef:794] [02:21:25.100] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:21:25.100+0000] [.info] [src/director/director.go:tracef:794] [02:21:25.100] because [2021-10-22T02:21:25.100+0000] [.info] [src/director/director.go:tracef:794] [02:21:25.100] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:21:25.100+0000] [.info] [src/director/director.go:tracef:794] [02:21:25.100] because ['currentState.CanBounceStop' = true] [2021-10-22T02:21:25.252+0000] [.info] [src/director/director.go:tracef:794] [02:21:25.252] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:21:25.252+0000] [.info] [src/director/director.go:tracef:794] [02:21:25.252] because [2021-10-22T02:21:25.252+0000] [.info] [src/director/director.go:tracef:794] [02:21:25.252] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:21:25.252+0000] [.info] [src/director/director.go:tracef:794] [02:21:25.252] because ['currentState.Up' = false] [2021-10-22T02:21:25.252+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:21:25.252] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:21:25.252+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:21:25.252] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:21:25.334+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:21:25.334] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:21:25.343+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:21:25.343] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:21:25.343+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:21:25.343] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:21:25.344+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:21:25.344] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:21:25.345+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:21:25.345] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:21:25.345+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:21:25.345] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:21:25.345+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:21:25.345] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:21:25.347+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:21:25.347] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:21:25.497+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:25.497] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:25.557+0000] [.info] [src/director/director.go:tracef:794] [02:21:25.557] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:21:25.557+0000] [.info] [src/director/director.go:tracef:794] [02:21:25.557] because ['currentState.Up' = false] [2021-10-22T02:21:25.754+0000] [.info] [src/director/director.go:tracef:794] [02:21:25.754] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:21:25.754+0000] [.info] [src/director/director.go:tracef:794] [02:21:25.754] because ['currentState.Up' = false] [2021-10-22T02:21:25.754+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:21:25.754] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:21:25.990+0000] [.info] [src/director/director.go:tracef:794] [02:21:25.990] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:21:25.990+0000] [.info] [src/director/director.go:tracef:794] [02:21:25.990] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:21:26.198+0000] [.info] [src/director/director.go:tracef:794] [02:21:26.198] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:21:26.198+0000] [.info] [src/director/director.go:tracef:794] [02:21:26.198] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:21:26.199+0000] [.info] [src/director/director.go:tracef:794] [02:21:26.198] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:21:26.199+0000] [.info] [src/director/director.go:tracef:794] [02:21:26.199] because ['currentState.Up' = false] [2021-10-22T02:21:26.199+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:21:26.199] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:21:26.199+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:21:26.199] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:21:26.232+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:21:26.232] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:21:26.233+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:21:26.233] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56300 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:21:26.491+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:26.491] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:26.792+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:21:26.792] clusterConfig unchanged [2021-10-22T02:21:26.952+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:21:26.952] ...Finished start [pid=62197] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56300,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:21:27.132+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:21:27.132] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 180.607831ms. [2021-10-22T02:21:27.132+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:21:27.132] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:56300 (local=false) [2021-10-22T02:21:27.132+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:21:27.132] Dropping existing documents from system.replset [2021-10-22T02:21:27.297+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:21:27.297] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:21:27.301+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:21:27.301] Inserting oplogTruncateAfterPoint document [2021-10-22T02:21:27.301+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:21:27.301] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:21:27.302+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:21:27.302] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:21:27.302+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:21:27.302] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:21:27.315+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:21:27.315] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:21:27.315+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:21:27.315] Setting the minvalid document to [{_id ObjectID("61722027523370002becc835")} {t -1} {ts {0 1}}] [2021-10-22T02:21:27.329+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:21:27.329] Inserted [{_id ObjectID("61722027523370002becc835")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:21:27.329+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:21:27.329] Inserting new replica set configuration document. [2021-10-22T02:21:27.330+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:21:27.330] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:21:27.343+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:21:27.343] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:21:27.348+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:21:27.348] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:21:27.348+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:21:27.348] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:21:27.348+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:21:27.348] Initiating stopEphemeralMongo [2021-10-22T02:21:27.348+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:21:27.348] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:56300 (local=false) [2021-10-22T02:21:27.348+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:21:27.348] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:56300 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:21:27.493+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:27.493] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:27.745+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:21:27.745] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.783957ms. [2021-10-22T02:21:27.745+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:21:27.745] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:56300 (local=false) [2021-10-22T02:21:27.745+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:21:27.745] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 40.957µs. [2021-10-22T02:21:27.745+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:21:27.745] Successfully finished running stopEphemeralMongo [2021-10-22T02:21:27.745+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:21:27.745] Successfully stopped existing ephemeral mongod. [2021-10-22T02:21:27.745+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:21:27.745] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:21:27.745+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:21:27.745] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:21:27.795+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:21:27.795] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:21:27.796+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:21:27.796] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45167 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:21:27.925+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:21:27.925] clusterConfig unchanged [2021-10-22T02:21:28.494+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:28.494] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:28.973+0000] [.error] [src/action/start.go:func1:93] [103] [02:21:28.973] Error starting mongod : [02:21:28.973] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45167,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45167 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 63684 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:21:27.848+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:21:27.848+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:21:27.848+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45167,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:21:27.850+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:21:27.852+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:21:27.854+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:21:27.880+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:21:27.880+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:21:27.893+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:21:27.937+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:21:27.937+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:21:28.449+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869288:449699][63684:0x7f326509ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:21:28.549+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869288:549312][63684:0x7f326509ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:21:28.667+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869288:667326][63684:0x7f326509ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:21:28.789+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869288:789125][63684:0x7f326509ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:21:28.861+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869288:861379][63684:0x7f326509ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:21:28.920+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869288:920373][63684:0x7f326509ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:21:28.920+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869288:920441][63684:0x7f326509ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:21:28.922+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:21:28.973+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:21:28.973] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:45167 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:21:28.973] Error starting mongod : [02:21:28.973] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45167,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45167 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 63684 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:21:27.848+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:21:27.848+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:21:27.848+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45167,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:21:27.850+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:21:27.852+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:21:27.854+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:21:27.880+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:21:27.880+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:21:27.893+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:21:27.937+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:21:27.937+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:21:28.449+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869288:449699][63684:0x7f326509ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:21:28.549+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869288:549312][63684:0x7f326509ebc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:21:28.973+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:21:28.973] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:21:28.973] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:45167 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:21:28.973] Error starting mongod : [02:21:28.973] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45167,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45167 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 63684 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:21:27.848+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:21:27.848+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:21:27.848+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45167,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:21:27.850+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:21:27.852+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:21:27.854+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:21:27.880+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:21:27.880+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:21:27.893+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:21:27.937+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:21:27.937+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:21:28.449+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869288:449699][63684:0x7f326509ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:21:28.549+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:21:28.973+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:21:28.973] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:21:28.973+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:21:28.973] Initiating stopEphemeralMongo [2021-10-22T02:21:28.973+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:21:28.973] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:21:28.974+0000] [.error] [src/director/director.go:executePlan:962] [02:21:28.974] Failed to apply action. Result = : [02:21:28.973] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:21:28.973] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:45167 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:21:28.973] Error starting mongod : [02:21:28.973] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45167,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45167 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 63684 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:21:27.848+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:21:27.848+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:21:27.848+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45167,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:21:27.850+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:21:27.852+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:21:27.854+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:21:27.880+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:21:27.880+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:21:27.893+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:21:27.937+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:21:27.937+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:21:28.449+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869288:449699][63684:0x7f326509ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:21:28.974+0000] [.error] [src/director/director.go:planAndExecute:567] [02:21:28.974] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:21:28.974] Failed to apply action. Result = : [02:21:28.973] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:21:28.973] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:45167 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:21:28.973] Error starting mongod : [02:21:28.973] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45167,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45167 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 63684 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:21:27.848+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:21:27.848+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:21:27.848+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45167,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:21:27.850+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:21:27.852+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:21:27.854+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:21:27.880+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:21:27.880+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:21:27.893+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:21:27.937+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:21:27.937+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:21:28.449+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:21:28.974+0000] [.error] [src/director/director.go:mainLoop:399] [02:21:28.974] Failed to planAndExecute : [02:21:28.974] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:21:28.974] Failed to apply action. Result = : [02:21:28.973] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:21:28.973] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:45167 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:21:28.973] Error starting mongod : [02:21:28.973] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45167,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45167 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 63684 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:21:27.848+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:21:27.848+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:21:27.848+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45167,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:21:27.850+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:21:27.852+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:21:27.854+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:21:27.880+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:21:27.880+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:21:27.893+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:21:27.937+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:21:27.937+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:21:28.449+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:21:29.036+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:21:29.036] clusterConfig unchanged [2021-10-22T02:21:29.494+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:29.494] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:30.267+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:21:30.267] clusterConfig unchanged [2021-10-22T02:21:30.494+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:30.494] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:31.495+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:31.495] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:32.502+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:32.502] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:33.493+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:33.493] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:34.496+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:34.496] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:35.495+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:35.495] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:36.500+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:36.500] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:37.499+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:37.499] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:38.107+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:21:38.107] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.759893642s [2021-10-22T02:21:38.500+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:38.500] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:39.499+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:39.499] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:40.371+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:21:40.371] clusterConfig unchanged [2021-10-22T02:21:40.499+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:40.499] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:41.395+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:21:41.395] memory:rss=59072512 alloc=14796760 totalalloc=10432106720 mallocs=133102124 frees=132971594 heapinuse=18644992 heapobjects=130530 gcpausetotalns=1629547429 gcpauselastns=111275 gcnum=1536 utimens=224790205000 stimens=88636028000 minflt=558480 majflt=405 [2021-10-22T02:21:41.500+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:41.500] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:42.509+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:42.508] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:43.501+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:43.501] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:44.501+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:44.501] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:45.501+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:45.501] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:46.501+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:46.501] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:47.502+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:47.502] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:48.503+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:48.503] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:49.503+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:49.503] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:50.484+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:21:50.484] clusterConfig unchanged [2021-10-22T02:21:50.503+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:50.503] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:51.504+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:51.504] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:52.504+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:52.504] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:53.504+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:53.504] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:54.508+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:54.508] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:55.513+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:55.513] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:56.505+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:56.505] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:57.506+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:57.506] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:58.506+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:58.506] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:21:59.507+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:21:59.507] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:00.507+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:00.507] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:00.604+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:22:00.604] clusterConfig unchanged [2021-10-22T02:22:01.508+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:01.508] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:02.508+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:02.508] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:03.509+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:03.509] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:04.509+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:04.509] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:05.510+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:05.510] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:06.519+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:06.519] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:07.512+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:07.512] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:08.514+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:08.514] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:09.514+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:09.514] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:10.515+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:10.515] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:10.706+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:22:10.706] clusterConfig unchanged [2021-10-22T02:22:11.516+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:11.516] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:12.516+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:12.516] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:13.516+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:13.516] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:14.517+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:14.517] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:15.533+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:15.533] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:16.518+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:16.518] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:17.525+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:17.525] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:18.520+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:18.520] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:19.520+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:19.520] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:20.521+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:20.521] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:20.809+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:22:20.809] clusterConfig unchanged [2021-10-22T02:22:21.520+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:21.520] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:22.521+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:22.521] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:23.521+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:23.521] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:24.521+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:24.521] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:25.523+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:25.523] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:26.523+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:26.523] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:27.524+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:27.524] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:27.570+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:22:27.570] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:22:27.571+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:22:27.571] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:22:28.524+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:28.524] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:29.147+0000] [.info] [src/director/director.go:computePlan:280] [02:22:29.147] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:22:29.148+0000] [.info] [src/director/director.go:tracef:794] [02:22:29.148] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:22:29.148+0000] [.info] [src/director/director.go:tracef:794] [02:22:29.148] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:22:29.148+0000] [.info] [src/director/director.go:tracef:794] [02:22:29.148] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:22:29.148+0000] [.info] [src/director/director.go:tracef:794] [02:22:29.148] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:22:29.148+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:22:29.148] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:22:29.302+0000] [.info] [src/director/director.go:tracef:794] [02:22:29.302] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:22:29.302+0000] [.info] [src/director/director.go:tracef:794] [02:22:29.302] because [2021-10-22T02:22:29.302+0000] [.info] [src/director/director.go:tracef:794] [02:22:29.302] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:22:29.302+0000] [.info] [src/director/director.go:tracef:794] [02:22:29.302] because ['currentState.CanBounceStop' = true] [2021-10-22T02:22:29.464+0000] [.info] [src/director/director.go:tracef:794] [02:22:29.464] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:22:29.464+0000] [.info] [src/director/director.go:tracef:794] [02:22:29.464] because [2021-10-22T02:22:29.464+0000] [.info] [src/director/director.go:tracef:794] [02:22:29.464] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:22:29.464+0000] [.info] [src/director/director.go:tracef:794] [02:22:29.464] because ['currentState.Up' = false] [2021-10-22T02:22:29.464+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:22:29.464] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:22:29.464+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:22:29.464] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:22:29.525+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:29.525] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:29.543+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:22:29.543] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:22:29.543+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:22:29.543] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:22:29.551+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:22:29.551] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:22:29.552+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:22:29.552] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:22:29.553+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:22:29.553] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:22:29.553+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:22:29.553] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:22:29.554+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:22:29.554] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:22:29.555+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:22:29.555] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:22:29.764+0000] [.info] [src/director/director.go:tracef:794] [02:22:29.764] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:22:29.764+0000] [.info] [src/director/director.go:tracef:794] [02:22:29.764] because ['currentState.Up' = false] [2021-10-22T02:22:29.978+0000] [.info] [src/director/director.go:tracef:794] [02:22:29.978] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:22:29.978+0000] [.info] [src/director/director.go:tracef:794] [02:22:29.978] because ['currentState.Up' = false] [2021-10-22T02:22:29.978+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:22:29.978] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:22:30.190+0000] [.info] [src/director/director.go:tracef:794] [02:22:30.190] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:22:30.190+0000] [.info] [src/director/director.go:tracef:794] [02:22:30.190] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:22:30.422+0000] [.info] [src/director/director.go:tracef:794] [02:22:30.422] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:22:30.422+0000] [.info] [src/director/director.go:tracef:794] [02:22:30.422] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:22:30.422+0000] [.info] [src/director/director.go:tracef:794] [02:22:30.422] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:22:30.422+0000] [.info] [src/director/director.go:tracef:794] [02:22:30.422] because ['currentState.Up' = false] [2021-10-22T02:22:30.422+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:22:30.422] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:22:30.422+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:22:30.422] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:22:30.456+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:22:30.456] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:22:30.457+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:22:30.457] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39079 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:22:30.529+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:30.529] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:31.030+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:22:31.030] clusterConfig unchanged [2021-10-22T02:22:31.184+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:22:31.184] ...Finished start [pid=8930] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39079,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:22:31.319+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:22:31.319] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 135.245451ms. [2021-10-22T02:22:31.319+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:22:31.319] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39079 (local=false) [2021-10-22T02:22:31.319+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:22:31.319] Dropping existing documents from system.replset [2021-10-22T02:22:31.440+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:22:31.440] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:22:31.445+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:22:31.445] Inserting oplogTruncateAfterPoint document [2021-10-22T02:22:31.445+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:22:31.445] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:22:31.445+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:22:31.445] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:22:31.446+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:22:31.446] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:22:31.457+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:22:31.457] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:22:31.458+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:22:31.458] Setting the minvalid document to [{_id ObjectID("61722067523370002becc843")} {t -1} {ts {0 1}}] [2021-10-22T02:22:31.471+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:22:31.470] Inserted [{_id ObjectID("61722067523370002becc843")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:22:31.471+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:22:31.471] Inserting new replica set configuration document. [2021-10-22T02:22:31.471+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:22:31.471] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:22:31.482+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:22:31.482] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:22:31.496+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:22:31.496] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:22:31.496+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:22:31.496] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:22:31.496+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:22:31.496] Initiating stopEphemeralMongo [2021-10-22T02:22:31.496+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:22:31.496] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39079 (local=false) [2021-10-22T02:22:31.496+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:22:31.496] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39079 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:22:31.537+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:31.537] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:31.929+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:22:31.929] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.743848ms. [2021-10-22T02:22:31.929+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:22:31.929] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39079 (local=false) [2021-10-22T02:22:31.930+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:22:31.930] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.918µs. [2021-10-22T02:22:31.930+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:22:31.930] Successfully finished running stopEphemeralMongo [2021-10-22T02:22:31.930+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:22:31.930] Successfully stopped existing ephemeral mongod. [2021-10-22T02:22:31.930+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:22:31.930] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:22:31.930+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:22:31.930] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:22:31.963+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:22:31.963] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:22:31.964+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:22:31.964] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51757 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:22:32.162+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:22:32.162] clusterConfig unchanged [2021-10-22T02:22:32.532+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:32.532] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:33.260+0000] [.error] [src/action/start.go:func1:93] [103] [02:22:33.260] Error starting mongod : [02:22:33.260] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51757,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51757 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 10385 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:22:32.015+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:22:32.015+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:22:32.015+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51757,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:22:32.017+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:22:32.019+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:22:32.020+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:22:32.031+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:22:32.031+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:22:32.031+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:22:32.077+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:22:32.077+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:22:32.642+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869352:642509][10385:0x7f1b4d06bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:22:32.808+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869352:808095][10385:0x7f1b4d06bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:22:32.940+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869352:940360][10385:0x7f1b4d06bbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35456 to 2/256"}} {"t":{"$date":"2021-10-22T02:22:33.059+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869353:59590][10385:0x7f1b4d06bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:22:33.138+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869353:138878][10385:0x7f1b4d06bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:22:33.201+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869353:201028][10385:0x7f1b4d06bbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:22:33.201+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869353:201107][10385:0x7f1b4d06bbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:22:33.203+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:22:33.260+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:22:33.260] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51757 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:22:33.260] Error starting mongod : [02:22:33.260] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51757,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51757 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 10385 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:22:32.015+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:22:32.015+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:22:32.015+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51757,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:22:32.017+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:22:32.019+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:22:32.020+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:22:32.031+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:22:32.031+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:22:32.031+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:22:32.077+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:22:32.077+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:22:32.642+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869352:642509][10385:0x7f1b4d06bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:22:32.808+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869352:808095][10385:0x7f1b4d06bbc0], txn-recover: [W...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:22:33.261+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:22:33.261] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:22:33.260] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51757 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:22:33.260] Error starting mongod : [02:22:33.260] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51757,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51757 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 10385 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:22:32.015+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:22:32.015+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:22:32.015+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51757,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:22:32.017+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:22:32.019+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:22:32.020+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:22:32.031+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:22:32.031+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:22:32.031+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:22:32.077+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:22:32.077+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:22:32.642+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869352:642509][10385:0x7f1b4d06bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:22:32.808+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ct... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:22:33.261+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:22:33.261] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:22:33.261+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:22:33.261] Initiating stopEphemeralMongo [2021-10-22T02:22:33.261+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:22:33.261] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:22:33.261+0000] [.error] [src/director/director.go:executePlan:962] [02:22:33.261] Failed to apply action. Result = : [02:22:33.261] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:22:33.260] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51757 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:22:33.260] Error starting mongod : [02:22:33.260] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51757,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51757 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 10385 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:22:32.015+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:22:32.015+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:22:32.015+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51757,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:22:32.017+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:22:32.019+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:22:32.020+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:22:32.031+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:22:32.031+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:22:32.031+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:22:32.077+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:22:32.077+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:22:32.642+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869352:642509][10385:0x7f1b4d06bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:22:33.262+0000] [.error] [src/director/director.go:planAndExecute:567] [02:22:33.262] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:22:33.261] Failed to apply action. Result = : [02:22:33.261] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:22:33.260] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51757 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:22:33.260] Error starting mongod : [02:22:33.260] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51757,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51757 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 10385 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:22:32.015+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:22:32.015+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:22:32.015+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51757,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:22:32.017+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:22:32.019+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:22:32.020+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:22:32.031+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:22:32.031+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:22:32.031+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:22:32.077+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:22:32.077+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:22:32.642+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:22:33.262+0000] [.error] [src/director/director.go:mainLoop:399] [02:22:33.262] Failed to planAndExecute : [02:22:33.262] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:22:33.261] Failed to apply action. Result = : [02:22:33.261] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:22:33.260] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:51757 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:22:33.260] Error starting mongod : [02:22:33.260] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51757,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51757 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 10385 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:22:32.015+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:22:32.015+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:22:32.015+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51757,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:22:32.017+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:22:32.019+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:22:32.020+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:22:32.031+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:22:32.031+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:22:32.031+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:22:32.077+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:22:32.077+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:22:32.642+00:00"},"s":"I", "c":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:22:33.363+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:22:33.363] clusterConfig unchanged [2021-10-22T02:22:33.532+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:33.532] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:34.531+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:34.531] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:34.593+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:22:34.593] clusterConfig unchanged [2021-10-22T02:22:35.533+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:35.533] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:36.535+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:36.535] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:37.535+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:37.535] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:38.538+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [02:22:38.538] lsblk cache invalidated, running lsblk command [2021-10-22T02:22:38.540+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:38.540] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:39.535+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:39.535] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:40.535+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:40.535] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:41.395+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:22:41.395] memory:rss=57049088 alloc=13138088 totalalloc=10524121520 mallocs=134290542 frees=134187304 heapinuse=18964480 heapobjects=103238 gcpausetotalns=1632961299 gcpauselastns=102542 gcnum=1548 utimens=227003193000 stimens=89675609000 minflt=563369 majflt=405 [2021-10-22T02:22:41.539+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:41.539] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:42.280+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:22:42.280] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.724783001s [2021-10-22T02:22:42.535+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:42.535] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:43.535+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:43.535] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:44.536+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:44.536] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:44.734+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:22:44.734] clusterConfig unchanged [2021-10-22T02:22:45.537+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:45.537] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:46.537+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:46.537] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:47.538+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:47.538] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:48.539+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:48.539] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:49.539+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:49.539] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:50.540+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:50.540] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:51.541+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:51.541] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:52.541+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:52.541] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:53.542+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:53.542] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:54.543+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:54.543] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:54.856+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:22:54.856] clusterConfig unchanged [2021-10-22T02:22:55.550+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:55.550] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:56.543+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:56.543] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:57.546+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:57.546] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:58.545+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:58.545] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:22:59.544+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:22:59.544] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:00.545+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:00.545] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:01.546+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:01.546] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:02.547+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:02.547] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:03.546+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:03.546] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:04.547+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:04.547] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:05.017+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:23:05.017] clusterConfig unchanged [2021-10-22T02:23:05.547+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:05.547] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:06.555+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:06.555] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:07.548+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:07.548] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:08.549+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:08.549] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:09.550+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:09.550] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:10.551+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:10.551] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:11.550+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:11.550] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:12.551+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:12.551] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:13.552+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:13.552] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:14.553+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:14.553] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:15.129+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:23:15.129] clusterConfig unchanged [2021-10-22T02:23:15.554+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:15.554] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:16.554+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:16.554] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:17.561+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:17.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:18.555+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:18.555] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:19.555+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:19.555] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:20.556+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:20.556] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:21.557+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:21.557] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:22.557+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:22.557] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:23.558+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:23.558] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:24.559+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:24.558] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:25.231+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:23:25.231] clusterConfig unchanged [2021-10-22T02:23:25.559+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:25.559] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:26.559+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:26.559] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:27.561+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:27.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:28.561+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:28.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:29.566+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:29.566] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:30.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:30.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:31.563+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:31.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:32.565+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:32.565] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:33.444+0000] [.info] [src/director/director.go:computePlan:280] [02:23:33.444] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:23:33.445+0000] [.info] [src/director/director.go:tracef:794] [02:23:33.445] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:23:33.445+0000] [.info] [src/director/director.go:tracef:794] [02:23:33.445] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:23:33.445+0000] [.info] [src/director/director.go:tracef:794] [02:23:33.445] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:23:33.445+0000] [.info] [src/director/director.go:tracef:794] [02:23:33.445] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:23:33.445+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:23:33.445] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:23:33.567+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:33.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:33.619+0000] [.info] [src/director/director.go:tracef:794] [02:23:33.619] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:23:33.619+0000] [.info] [src/director/director.go:tracef:794] [02:23:33.619] because [2021-10-22T02:23:33.619+0000] [.info] [src/director/director.go:tracef:794] [02:23:33.619] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:23:33.619+0000] [.info] [src/director/director.go:tracef:794] [02:23:33.619] because ['currentState.CanBounceStop' = true] [2021-10-22T02:23:33.773+0000] [.info] [src/director/director.go:tracef:794] [02:23:33.773] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:23:33.773+0000] [.info] [src/director/director.go:tracef:794] [02:23:33.773] because [2021-10-22T02:23:33.773+0000] [.info] [src/director/director.go:tracef:794] [02:23:33.773] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:23:33.773+0000] [.info] [src/director/director.go:tracef:794] [02:23:33.773] because ['currentState.Up' = false] [2021-10-22T02:23:33.773+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:23:33.773] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:23:33.773+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:23:33.773] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:23:33.876+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:23:33.876] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:23:33.884+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:23:33.884] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:23:33.885+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:23:33.885] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:23:33.886+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:23:33.886] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:23:33.886+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:23:33.886] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:23:33.886+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:23:33.886] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:23:33.888+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:23:33.888] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:23:34.123+0000] [.info] [src/director/director.go:tracef:794] [02:23:34.123] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:23:34.123+0000] [.info] [src/director/director.go:tracef:794] [02:23:34.123] because ['currentState.Up' = false] [2021-10-22T02:23:34.358+0000] [.info] [src/director/director.go:tracef:794] [02:23:34.358] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:23:34.359+0000] [.info] [src/director/director.go:tracef:794] [02:23:34.359] because ['currentState.Up' = false] [2021-10-22T02:23:34.359+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:23:34.359] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:23:34.568+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:34.568] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:34.579+0000] [.info] [src/director/director.go:tracef:794] [02:23:34.579] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:23:34.579+0000] [.info] [src/director/director.go:tracef:794] [02:23:34.579] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:23:34.806+0000] [.info] [src/director/director.go:tracef:794] [02:23:34.806] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:23:34.806+0000] [.info] [src/director/director.go:tracef:794] [02:23:34.806] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:23:34.807+0000] [.info] [src/director/director.go:tracef:794] [02:23:34.807] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:23:34.807+0000] [.info] [src/director/director.go:tracef:794] [02:23:34.807] because ['currentState.Up' = false] [2021-10-22T02:23:34.807+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:23:34.807] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:23:34.807+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:23:34.807] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:23:34.844+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:23:34.844] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:23:34.844+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:23:34.844] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42126 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:23:35.501+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:23:35.501] clusterConfig unchanged [2021-10-22T02:23:35.569+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:35.569] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:35.575+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:23:35.575] ...Finished start [pid=29882] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42126,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:23:35.709+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:23:35.709] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 133.857007ms. [2021-10-22T02:23:35.709+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:23:35.709] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42126 (local=false) [2021-10-22T02:23:35.709+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:23:35.709] Dropping existing documents from system.replset [2021-10-22T02:23:35.831+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:23:35.831] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:23:35.835+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:23:35.835] Inserting oplogTruncateAfterPoint document [2021-10-22T02:23:35.835+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:23:35.835] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:23:35.836+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:23:35.836] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:23:35.836+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:23:35.836] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:23:35.850+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:23:35.850] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:23:35.850+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:23:35.850] Setting the minvalid document to [{_id ObjectID("617220a7523370002becc851")} {t -1} {ts {0 1}}] [2021-10-22T02:23:35.863+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:23:35.863] Inserted [{_id ObjectID("617220a7523370002becc851")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:23:35.863+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:23:35.863] Inserting new replica set configuration document. [2021-10-22T02:23:35.863+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:23:35.863] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:23:35.875+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:23:35.875] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:23:35.878+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:23:35.878] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:23:35.878+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:23:35.878] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:23:35.878+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:23:35.878] Initiating stopEphemeralMongo [2021-10-22T02:23:35.878+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:23:35.878] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42126 (local=false) [2021-10-22T02:23:35.878+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:23:35.878] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42126 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:23:36.291+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:23:36.291] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.122467ms. [2021-10-22T02:23:36.291+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:23:36.291] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42126 (local=false) [2021-10-22T02:23:36.291+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:23:36.291] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 38.22µs. [2021-10-22T02:23:36.291+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:23:36.291] Successfully finished running stopEphemeralMongo [2021-10-22T02:23:36.291+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:23:36.291] Successfully stopped existing ephemeral mongod. [2021-10-22T02:23:36.291+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:23:36.291] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:23:36.291+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:23:36.291] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:23:36.323+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:23:36.323] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:23:36.323+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:23:36.323] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42330 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:23:36.570+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:36.570] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:36.634+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:23:36.634] clusterConfig unchanged [2021-10-22T02:23:37.531+0000] [.error] [src/action/start.go:func1:93] [103] [02:23:37.531] Error starting mongod : [02:23:37.531] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42330,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42330 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:23:36.357Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 31410 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:23:36.381+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:23:36.381+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:23:36.381+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42330,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:23:36.383+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:23:36.386+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:23:36.386+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:23:36.430+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:23:36.430+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:23:36.430+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:23:36.486+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:23:36.487+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:23:37.016+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869417:16940][31410:0x7f95389c8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:23:37.128+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869417:128667][31410:0x7f95389c8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:23:37.236+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869417:236025][31410:0x7f95389c8bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T02:23:37.355+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869417:355331][31410:0x7f95389c8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:23:37.428+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869417:428613][31410:0x7f95389c8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:23:37.485+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869417:485571][31410:0x7f95389c8bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:23:37.485+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869417:485645][31410:0x7f95389c8bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:23:37.532+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:23:37.532] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42330 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:23:37.531] Error starting mongod : [02:23:37.531] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42330,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42330 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:23:36.357Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 31410 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:23:36.381+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:23:36.381+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:23:36.381+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42330,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:23:36.383+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:23:36.386+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:23:36.386+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:23:36.430+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:23:36.430+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:23:36.430+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:23:36.486+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:23:36.487+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:23:37.016+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869417:16940][31410:0x7f95389c8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:23:37.128+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:23:37.532+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:23:37.532] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:23:37.532] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42330 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:23:37.531] Error starting mongod : [02:23:37.531] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42330,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42330 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:23:36.357Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 31410 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:23:36.381+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:23:36.381+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:23:36.381+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42330,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:23:36.383+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:23:36.386+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:23:36.386+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:23:36.430+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:23:36.430+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:23:36.430+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:23:36.486+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:23:36.487+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:23:37.016+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869417:16940][31410:0x7f95389c8bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:23:37.532+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:23:37.532] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:23:37.532+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:23:37.532] Initiating stopEphemeralMongo [2021-10-22T02:23:37.532+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:23:37.532] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:23:37.533+0000] [.error] [src/director/director.go:executePlan:962] [02:23:37.533] Failed to apply action. Result = : [02:23:37.532] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:23:37.532] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42330 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:23:37.531] Error starting mongod : [02:23:37.531] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42330,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42330 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:23:36.357Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 31410 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:23:36.381+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:23:36.381+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:23:36.381+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42330,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:23:36.383+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:23:36.386+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:23:36.386+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:23:36.430+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:23:36.430+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:23:36.430+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:23:36.486+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:23:36.487+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:23:37.016+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:23:37.533+0000] [.error] [src/director/director.go:planAndExecute:567] [02:23:37.533] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:23:37.533] Failed to apply action. Result = : [02:23:37.532] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:23:37.532] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42330 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:23:37.531] Error starting mongod : [02:23:37.531] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42330,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42330 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:23:36.357Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 31410 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:23:36.381+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:23:36.381+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:23:36.381+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42330,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:23:36.383+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:23:36.386+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:23:36.386+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:23:36.430+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:23:36.430+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:23:36.430+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:23:36.486+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:23:36.487+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:23:37.533+0000] [.error] [src/director/director.go:mainLoop:399] [02:23:37.533] Failed to planAndExecute : [02:23:37.533] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:23:37.533] Failed to apply action. Result = : [02:23:37.532] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:23:37.532] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42330 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:23:37.531] Error starting mongod : [02:23:37.531] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42330,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42330 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:23:36.357Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 31410 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:23:36.381+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:23:36.381+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:23:36.381+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42330,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:23:36.383+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:23:36.386+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:23:36.386+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:23:36.430+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:23:36.430+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:23:36.430+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:23:36.486+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:23:36.487+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:23:37.575+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:37.575] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:37.858+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:23:37.858] clusterConfig unchanged [2021-10-22T02:23:38.578+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:38.578] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:39.578+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:39.578] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:40.578+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:40.578] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:41.397+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:23:41.397] memory:rss=58261504 alloc=9804856 totalalloc=10615359232 mallocs=135473939 frees=135401056 heapinuse=17670144 heapobjects=72883 gcpausetotalns=1634251447 gcpauselastns=103335 gcnum=1560 utimens=229250024000 stimens=90583000000 minflt=568911 majflt=405 [2021-10-22T02:23:41.580+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:41.580] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:42.580+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:42.580] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:43.580+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:43.580] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:44.425+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [02:23:44.425] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42330 (local=false) because it is not up [2021-10-22T02:23:44.581+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:44.581] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:45.582+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:45.582] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:46.581+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:46.581] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:46.626+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:23:46.626] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.73835411s [2021-10-22T02:23:47.584+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:47.584] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:47.967+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:23:47.967] clusterConfig unchanged [2021-10-22T02:23:48.597+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:48.597] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:49.582+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:49.582] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:50.583+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:50.583] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:51.587+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:51.587] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:52.584+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:52.584] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:53.584+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:53.584] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:54.585+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:54.585] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:55.586+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:55.586] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:56.587+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:56.587] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:57.587+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:57.587] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:58.157+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:23:58.157] clusterConfig unchanged [2021-10-22T02:23:58.587+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:58.587] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:23:59.595+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:23:59.595] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:00.588+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:00.588] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:01.589+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:01.589] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:02.590+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:02.590] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:03.591+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:03.591] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:04.591+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:04.591] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:05.592+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:05.592] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:06.593+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:06.593] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:07.593+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:07.593] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:08.250+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:24:08.250] clusterConfig unchanged [2021-10-22T02:24:08.593+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:08.593] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:09.600+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:09.600] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:10.594+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:10.594] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:11.595+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:11.595] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:12.595+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:12.595] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:13.596+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:13.596] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:14.596+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:14.596] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:15.597+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:15.596] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:16.597+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:16.597] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:17.598+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:17.598] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:18.381+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:24:18.381] clusterConfig unchanged [2021-10-22T02:24:18.599+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:18.599] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:19.599+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:19.599] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:20.604+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:20.604] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:21.600+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:21.600] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:22.601+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:22.601] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:23.601+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:23.601] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:24.602+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:24.602] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:25.603+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:25.603] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:26.604+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:26.604] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:27.605+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:27.605] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:28.513+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:24:28.513] clusterConfig unchanged [2021-10-22T02:24:28.605+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:28.605] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:29.607+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:29.607] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:30.607+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:30.607] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:31.607+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:31.607] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:32.608+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:32.608] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:33.608+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:33.608] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:34.609+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:34.609] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:35.609+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:35.609] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:36.610+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:36.610] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:37.613+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:37.613] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:37.703+0000] [.info] [src/director/director.go:computePlan:280] [02:24:37.703] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:24:37.703+0000] [.info] [src/director/director.go:tracef:794] [02:24:37.703] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:24:37.703+0000] [.info] [src/director/director.go:tracef:794] [02:24:37.703] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:24:37.703+0000] [.info] [src/director/director.go:tracef:794] [02:24:37.703] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:24:37.703+0000] [.info] [src/director/director.go:tracef:794] [02:24:37.703] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:24:37.703+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:24:37.703] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:24:37.878+0000] [.info] [src/director/director.go:tracef:794] [02:24:37.878] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:24:37.879+0000] [.info] [src/director/director.go:tracef:794] [02:24:37.879] because [2021-10-22T02:24:37.879+0000] [.info] [src/director/director.go:tracef:794] [02:24:37.879] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:24:37.879+0000] [.info] [src/director/director.go:tracef:794] [02:24:37.879] because ['currentState.CanBounceStop' = true] [2021-10-22T02:24:38.036+0000] [.info] [src/director/director.go:tracef:794] [02:24:38.036] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:24:38.036+0000] [.info] [src/director/director.go:tracef:794] [02:24:38.036] because [2021-10-22T02:24:38.036+0000] [.info] [src/director/director.go:tracef:794] [02:24:38.036] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:24:38.036+0000] [.info] [src/director/director.go:tracef:794] [02:24:38.036] because ['currentState.Up' = false] [2021-10-22T02:24:38.036+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:24:38.036] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:24:38.036+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:24:38.036] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:24:38.182+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:24:38.182] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:24:38.190+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:24:38.190] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:24:38.190+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:24:38.190] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:24:38.191+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:24:38.191] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:24:38.192+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:24:38.192] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:24:38.192+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:24:38.192] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:24:38.193+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:24:38.193] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:24:38.194+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:24:38.194] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:24:38.428+0000] [.info] [src/director/director.go:tracef:794] [02:24:38.428] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:24:38.428+0000] [.info] [src/director/director.go:tracef:794] [02:24:38.428] because ['currentState.Up' = false] [2021-10-22T02:24:38.616+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:38.616] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:38.650+0000] [.info] [src/director/director.go:tracef:794] [02:24:38.650] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:24:38.650+0000] [.info] [src/director/director.go:tracef:794] [02:24:38.650] because ['currentState.Up' = false] [2021-10-22T02:24:38.650+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:24:38.650] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:24:38.726+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:24:38.726] clusterConfig unchanged [2021-10-22T02:24:38.852+0000] [.info] [src/director/director.go:tracef:794] [02:24:38.852] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:24:38.853+0000] [.info] [src/director/director.go:tracef:794] [02:24:38.852] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:24:39.068+0000] [.info] [src/director/director.go:tracef:794] [02:24:39.068] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:24:39.068+0000] [.info] [src/director/director.go:tracef:794] [02:24:39.068] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:24:39.069+0000] [.info] [src/director/director.go:tracef:794] [02:24:39.069] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:24:39.069+0000] [.info] [src/director/director.go:tracef:794] [02:24:39.069] because ['currentState.Up' = false] [2021-10-22T02:24:39.069+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:24:39.069] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:24:39.069+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:24:39.069] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:24:39.102+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:24:39.102] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:24:39.103+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:24:39.103] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 32983 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:24:39.615+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:39.615] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:39.815+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:24:39.815] ...Finished start [pid=50504] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":32983,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:24:39.862+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:24:39.862] clusterConfig unchanged [2021-10-22T02:24:39.960+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:24:39.960] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 144.580808ms. [2021-10-22T02:24:39.960+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:24:39.960] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:32983 (local=false) [2021-10-22T02:24:39.960+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:24:39.960] Dropping existing documents from system.replset [2021-10-22T02:24:40.111+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:24:40.111] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:24:40.133+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:24:40.133] Inserting oplogTruncateAfterPoint document [2021-10-22T02:24:40.133+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:24:40.133] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:24:40.134+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:24:40.134] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:24:40.134+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:24:40.134] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:24:40.146+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:24:40.146] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:24:40.146+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:24:40.146] Setting the minvalid document to [{_id ObjectID("617220e8523370002becc85f")} {t -1} {ts {0 1}}] [2021-10-22T02:24:40.158+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:24:40.158] Inserted [{_id ObjectID("617220e8523370002becc85f")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:24:40.158+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:24:40.158] Inserting new replica set configuration document. [2021-10-22T02:24:40.158+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:24:40.158] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:24:40.181+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:24:40.181] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:24:40.185+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:24:40.185] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:24:40.185+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:24:40.185] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:24:40.185+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:24:40.185] Initiating stopEphemeralMongo [2021-10-22T02:24:40.185+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:24:40.185] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:32983 (local=false) [2021-10-22T02:24:40.185+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:24:40.185] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:32983 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:24:40.589+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:24:40.589] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.012396ms. [2021-10-22T02:24:40.589+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:24:40.589] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:32983 (local=false) [2021-10-22T02:24:40.589+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:24:40.589] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 38.477µs. [2021-10-22T02:24:40.590+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:24:40.589] Successfully finished running stopEphemeralMongo [2021-10-22T02:24:40.590+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:24:40.590] Successfully stopped existing ephemeral mongod. [2021-10-22T02:24:40.590+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:24:40.590] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:24:40.590+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:24:40.590] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:24:40.615+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:40.615] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:40.623+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:24:40.623] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:24:40.624+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:24:40.624] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55987 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:24:41.000+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:24:41.000] clusterConfig unchanged [2021-10-22T02:24:41.404+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:24:41.404] memory:rss=57405440 alloc=14231224 totalalloc=10704743976 mallocs=136656412 frees=136537097 heapinuse=18661376 heapobjects=119315 gcpausetotalns=1637559095 gcpauselastns=106875 gcnum=1571 utimens=231366006000 stimens=91459886000 minflt=573351 majflt=405 [2021-10-22T02:24:41.617+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:41.617] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:41.860+0000] [.error] [src/action/start.go:func1:93] [103] [02:24:41.860] Error starting mongod : [02:24:41.860] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55987,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55987 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:24:40.653Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 52135 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:24:40.685+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:24:40.685+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:24:40.685+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55987,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:24:40.686+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:24:40.688+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:24:40.689+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:24:40.732+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:24:40.732+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:24:40.733+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:24:40.789+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:24:40.789+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:24:41.300+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869481:300115][52135:0x7fd8847efbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:24:41.425+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869481:425255][52135:0x7fd8847efbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:24:41.523+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869481:523565][52135:0x7fd8847efbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35456 to 2/256"}} {"t":{"$date":"2021-10-22T02:24:41.646+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869481:646131][52135:0x7fd8847efbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:24:41.721+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869481:721396][52135:0x7fd8847efbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:24:41.786+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869481:786359][52135:0x7fd8847efbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:24:41.786+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869481:786412][52135:0x7fd8847efbc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:24:41.860+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:24:41.860] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:55987 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:24:41.860] Error starting mongod : [02:24:41.860] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55987,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55987 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:24:40.653Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 52135 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:24:40.685+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:24:40.685+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:24:40.685+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55987,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:24:40.686+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:24:40.688+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:24:40.689+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:24:40.732+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:24:40.732+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:24:40.733+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:24:40.789+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:24:40.789+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:24:41.300+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869481:300115][52135:0x7fd8847efbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:24:41.425+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:24:41.860+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:24:41.860] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:24:41.860] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:55987 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:24:41.860] Error starting mongod : [02:24:41.860] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55987,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55987 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:24:40.653Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 52135 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:24:40.685+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:24:40.685+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:24:40.685+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55987,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:24:40.686+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:24:40.688+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:24:40.689+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:24:40.732+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:24:40.732+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:24:40.733+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:24:40.789+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:24:40.789+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:24:41.300+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869481:300115][52135:0x7fd8847efbc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:24:41.861+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:24:41.861] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:24:41.861+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:24:41.861] Initiating stopEphemeralMongo [2021-10-22T02:24:41.861+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:24:41.861] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:24:41.861+0000] [.error] [src/director/director.go:executePlan:962] [02:24:41.861] Failed to apply action. Result = : [02:24:41.860] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:24:41.860] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:55987 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:24:41.860] Error starting mongod : [02:24:41.860] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55987,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55987 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:24:40.653Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 52135 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:24:40.685+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:24:40.685+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:24:40.685+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55987,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:24:40.686+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:24:40.688+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:24:40.689+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:24:40.732+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:24:40.732+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:24:40.733+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:24:40.789+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:24:40.789+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:24:41.300+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:24:41.861+0000] [.error] [src/director/director.go:planAndExecute:567] [02:24:41.861] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:24:41.861] Failed to apply action. Result = : [02:24:41.860] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:24:41.860] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:55987 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:24:41.860] Error starting mongod : [02:24:41.860] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55987,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55987 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:24:40.653Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 52135 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:24:40.685+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:24:40.685+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:24:40.685+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55987,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:24:40.686+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:24:40.688+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:24:40.689+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:24:40.732+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:24:40.732+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:24:40.733+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:24:40.789+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:24:40.789+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:24:41.861+0000] [.error] [src/director/director.go:mainLoop:399] [02:24:41.861] Failed to planAndExecute : [02:24:41.861] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:24:41.861] Failed to apply action. Result = : [02:24:41.860] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:24:41.860] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:55987 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:24:41.860] Error starting mongod : [02:24:41.860] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55987,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55987 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:24:40.653Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 52135 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:24:40.685+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:24:40.685+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:24:40.685+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55987,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:24:40.686+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:24:40.688+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:24:40.689+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:24:40.732+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:24:40.732+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:24:40.733+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:24:40.789+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:24:40.789+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:24:42.218+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:24:42.218] clusterConfig unchanged [2021-10-22T02:24:42.615+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:42.615] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:43.615+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:43.615] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:44.616+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:44.616] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:45.619+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:45.619] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:46.619+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:46.619] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:47.619+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:47.619] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:48.620+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:48.620] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:49.650+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:49.650] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:50.631+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:50.631] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:51.061+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:24:51.061] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.867398229s [2021-10-22T02:24:51.645+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:51.645] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:52.330+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:24:52.330] clusterConfig unchanged [2021-10-22T02:24:52.630+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:52.630] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:53.631+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:53.631] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:54.631+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:54.631] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:55.632+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:55.632] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:56.632+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:56.632] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:57.632+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:57.632] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:58.633+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:58.633] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:24:59.637+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:24:59.637] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:00.634+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:00.634] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:01.635+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:01.635] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:02.442+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:25:02.442] clusterConfig unchanged [2021-10-22T02:25:02.636+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:02.636] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:03.636+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:03.635] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:04.636+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:04.636] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:05.637+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:05.637] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:06.637+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:06.637] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:07.638+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:07.638] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:08.638+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:08.638] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:09.638+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:09.638] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:10.640+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:10.640] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:11.640+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:11.640] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:12.554+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:25:12.554] clusterConfig unchanged [2021-10-22T02:25:12.641+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:12.641] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:13.642+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:13.642] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:14.642+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:14.642] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:15.644+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:15.644] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:16.659+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:16.659] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:17.644+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:17.644] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:18.645+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:18.645] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:19.646+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:19.646] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:20.646+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:20.646] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:21.646+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:21.646] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:22.647+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:22.647] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:22.755+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:25:22.755] clusterConfig unchanged [2021-10-22T02:25:23.647+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:23.647] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:24.648+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:24.648] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:25.648+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:25.648] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:26.649+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:26.649] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:27.650+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:27.650] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:28.651+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:28.651] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:29.651+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:29.651] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:30.652+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:30.652] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:31.652+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:31.652] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:32.654+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:32.654] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:32.876+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:25:32.876] clusterConfig unchanged [2021-10-22T02:25:33.653+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:33.653] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:34.654+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:34.654] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:35.654+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:35.654] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:36.655+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:36.655] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:37.655+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:37.655] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:38.655+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:38.655] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:39.656+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:39.656] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:40.657+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:40.657] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:41.405+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:25:41.405] memory:rss=57024512 alloc=13334176 totalalloc=10748926880 mallocs=137179729 frees=137064697 heapinuse=17588224 heapobjects=115032 gcpausetotalns=1638044733 gcpauselastns=64634 gcnum=1577 utimens=233123625000 stimens=92420363000 minflt=577550 majflt=405 [2021-10-22T02:25:41.658+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:41.658] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:42.038+0000] [.info] [src/director/director.go:computePlan:280] [02:25:42.038] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:25:42.038+0000] [.info] [src/director/director.go:tracef:794] [02:25:42.038] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:25:42.038+0000] [.info] [src/director/director.go:tracef:794] [02:25:42.038] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:25:42.039+0000] [.info] [src/director/director.go:tracef:794] [02:25:42.039] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:25:42.039+0000] [.info] [src/director/director.go:tracef:794] [02:25:42.039] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:25:42.039+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:25:42.039] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:25:42.192+0000] [.info] [src/director/director.go:tracef:794] [02:25:42.192] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:25:42.192+0000] [.info] [src/director/director.go:tracef:794] [02:25:42.192] because [2021-10-22T02:25:42.192+0000] [.info] [src/director/director.go:tracef:794] [02:25:42.192] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:25:42.192+0000] [.info] [src/director/director.go:tracef:794] [02:25:42.192] because ['currentState.CanBounceStop' = true] [2021-10-22T02:25:42.351+0000] [.info] [src/director/director.go:tracef:794] [02:25:42.351] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:25:42.351+0000] [.info] [src/director/director.go:tracef:794] [02:25:42.351] because [2021-10-22T02:25:42.351+0000] [.info] [src/director/director.go:tracef:794] [02:25:42.351] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:25:42.351+0000] [.info] [src/director/director.go:tracef:794] [02:25:42.351] because ['currentState.Up' = false] [2021-10-22T02:25:42.352+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:25:42.352] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:25:42.352+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:25:42.352] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:25:42.428+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:25:42.428] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:25:42.436+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:25:42.436] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:25:42.437+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:25:42.437] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:25:42.437+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:25:42.437] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:25:42.438+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:25:42.438] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:25:42.438+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:25:42.438] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:25:42.438+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:25:42.438] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:25:42.440+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:25:42.440] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:25:42.643+0000] [.info] [src/director/director.go:tracef:794] [02:25:42.643] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:25:42.643+0000] [.info] [src/director/director.go:tracef:794] [02:25:42.643] because ['currentState.Up' = false] [2021-10-22T02:25:42.673+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:42.673] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:42.841+0000] [.info] [src/director/director.go:tracef:794] [02:25:42.841] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:25:42.841+0000] [.info] [src/director/director.go:tracef:794] [02:25:42.841] because ['currentState.Up' = false] [2021-10-22T02:25:42.841+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:25:42.841] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:25:43.058+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:25:43.057] clusterConfig unchanged [2021-10-22T02:25:43.074+0000] [.info] [src/director/director.go:tracef:794] [02:25:43.074] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:25:43.074+0000] [.info] [src/director/director.go:tracef:794] [02:25:43.074] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:25:43.281+0000] [.info] [src/director/director.go:tracef:794] [02:25:43.281] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:25:43.281+0000] [.info] [src/director/director.go:tracef:794] [02:25:43.281] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:25:43.281+0000] [.info] [src/director/director.go:tracef:794] [02:25:43.281] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:25:43.282+0000] [.info] [src/director/director.go:tracef:794] [02:25:43.281] because ['currentState.Up' = false] [2021-10-22T02:25:43.282+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:25:43.282] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:25:43.282+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:25:43.282] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:25:43.318+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:25:43.318] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:25:43.319+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:25:43.319] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47410 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:25:43.662+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:43.662] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:44.056+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:25:44.056] ...Finished start [pid=71213] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47410,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:25:44.204+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:25:44.204] clusterConfig unchanged [2021-10-22T02:25:44.209+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:25:44.208] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 152.087497ms. [2021-10-22T02:25:44.209+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:25:44.209] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47410 (local=false) [2021-10-22T02:25:44.209+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:25:44.209] Dropping existing documents from system.replset [2021-10-22T02:25:44.398+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:25:44.398] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:25:44.401+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:25:44.401] Inserting oplogTruncateAfterPoint document [2021-10-22T02:25:44.401+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:25:44.401] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:25:44.403+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:25:44.403] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:25:44.403+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:25:44.403] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:25:44.417+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:25:44.417] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:25:44.417+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:25:44.417] Setting the minvalid document to [{_id ObjectID("61722128523370002becc86d")} {t -1} {ts {0 1}}] [2021-10-22T02:25:44.431+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:25:44.431] Inserted [{_id ObjectID("61722128523370002becc86d")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:25:44.431+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:25:44.431] Inserting new replica set configuration document. [2021-10-22T02:25:44.431+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:25:44.431] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:25:44.445+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:25:44.445] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:25:44.459+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:25:44.459] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:25:44.459+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:25:44.459] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:25:44.459+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:25:44.459] Initiating stopEphemeralMongo [2021-10-22T02:25:44.459+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:25:44.459] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47410 (local=false) [2021-10-22T02:25:44.459+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:25:44.459] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47410 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:25:44.662+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:44.662] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:44.822+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:25:44.821] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.108648ms. [2021-10-22T02:25:44.822+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:25:44.822] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47410 (local=false) [2021-10-22T02:25:44.822+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:25:44.822] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.838µs. [2021-10-22T02:25:44.822+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:25:44.822] Successfully finished running stopEphemeralMongo [2021-10-22T02:25:44.822+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:25:44.822] Successfully stopped existing ephemeral mongod. [2021-10-22T02:25:44.822+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:25:44.822] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:25:44.822+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:25:44.822] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:25:44.853+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:25:44.853] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:25:44.856+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:25:44.856] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60760 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:25:45.353+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:25:45.353] clusterConfig unchanged [2021-10-22T02:25:45.662+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:45.662] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:46.031+0000] [.error] [src/action/start.go:func1:93] [103] [02:25:46.031] Error starting mongod : [02:25:46.031] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60760,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60760 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 72704 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:25:44.918+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:25:44.918+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:25:44.918+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60760,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:25:44.920+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:25:44.922+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:25:44.923+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:25:44.955+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:25:44.955+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:25:44.957+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:25:45.005+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:25:45.005+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:25:45.521+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869545:521730][72704:0x7f53b4dfdbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:25:45.623+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869545:623533][72704:0x7f53b4dfdbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:25:45.725+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869545:725527][72704:0x7f53b4dfdbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:25:45.842+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869545:842079][72704:0x7f53b4dfdbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:25:45.920+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869545:920802][72704:0x7f53b4dfdbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:25:45.982+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869545:982740][72704:0x7f53b4dfdbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:25:45.982+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869545:982820][72704:0x7f53b4dfdbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:25:45.984+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:25:46.032+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:25:46.031] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60760 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:25:46.031] Error starting mongod : [02:25:46.031] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60760,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60760 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 72704 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:25:44.918+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:25:44.918+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:25:44.918+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60760,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:25:44.920+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:25:44.922+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:25:44.923+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:25:44.955+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:25:44.955+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:25:44.957+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:25:45.005+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:25:45.005+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:25:45.521+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869545:521730][72704:0x7f53b4dfdbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:25:45.623+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869545:623533][72704:0x7f53b4dfdbc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:25:46.032+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:25:46.032] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:25:46.031] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60760 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:25:46.031] Error starting mongod : [02:25:46.031] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60760,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60760 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 72704 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:25:44.918+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:25:44.918+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:25:44.918+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60760,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:25:44.920+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:25:44.922+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:25:44.923+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:25:44.955+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:25:44.955+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:25:44.957+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:25:45.005+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:25:45.005+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:25:45.521+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869545:521730][72704:0x7f53b4dfdbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:25:45.623+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:25:46.032+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:25:46.032] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:25:46.032+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:25:46.032] Initiating stopEphemeralMongo [2021-10-22T02:25:46.032+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:25:46.032] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:25:46.033+0000] [.error] [src/director/director.go:executePlan:962] [02:25:46.033] Failed to apply action. Result = : [02:25:46.032] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:25:46.031] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60760 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:25:46.031] Error starting mongod : [02:25:46.031] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60760,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60760 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 72704 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:25:44.918+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:25:44.918+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:25:44.918+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60760,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:25:44.920+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:25:44.922+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:25:44.923+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:25:44.955+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:25:44.955+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:25:44.957+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:25:45.005+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:25:45.005+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:25:45.521+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869545:521730][72704:0x7f53b4dfdbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:25:46.033+0000] [.error] [src/director/director.go:planAndExecute:567] [02:25:46.033] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:25:46.033] Failed to apply action. Result = : [02:25:46.032] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:25:46.031] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60760 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:25:46.031] Error starting mongod : [02:25:46.031] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60760,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60760 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 72704 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:25:44.918+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:25:44.918+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:25:44.918+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60760,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:25:44.920+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:25:44.922+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:25:44.923+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:25:44.955+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:25:44.955+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:25:44.957+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:25:45.005+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:25:45.005+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:25:45.521+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:25:46.033+0000] [.error] [src/director/director.go:mainLoop:399] [02:25:46.033] Failed to planAndExecute : [02:25:46.033] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:25:46.033] Failed to apply action. Result = : [02:25:46.032] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:25:46.031] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60760 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:25:46.031] Error starting mongod : [02:25:46.031] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60760,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60760 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 72704 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:25:44.918+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:25:44.918+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:25:44.918+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60760,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:25:44.920+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:25:44.922+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:25:44.923+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:25:44.955+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:25:44.955+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:25:44.957+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:25:45.005+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:25:45.005+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:25:45.521+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:25:46.602+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:25:46.602] clusterConfig unchanged [2021-10-22T02:25:46.662+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:46.662] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:47.662+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:47.662] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:48.664+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:48.664] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:49.664+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:49.664] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:50.666+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:50.666] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:51.668+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:51.668] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:52.672+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:52.672] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:53.682+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:53.682] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:54.668+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:54.668] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:55.185+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:25:55.185] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.745377039s [2021-10-22T02:25:55.666+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:55.666] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:56.667+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:56.667] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:56.724+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:25:56.724] clusterConfig unchanged [2021-10-22T02:25:57.667+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:57.667] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:58.668+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:58.668] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:25:59.669+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:25:59.669] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:00.670+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:00.670] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:01.683+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:01.683] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:02.680+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:02.680] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:03.673+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:03.673] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:04.676+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:04.676] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:05.674+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:05.674] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:06.674+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:06.674] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:06.816+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:26:06.816] clusterConfig unchanged [2021-10-22T02:26:07.675+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:07.675] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:08.675+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:08.674] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:09.675+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:09.675] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:10.676+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:10.676] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:11.676+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:11.676] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:12.677+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:12.677] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:13.678+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:13.678] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:14.678+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:14.678] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:15.679+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:15.679] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:16.679+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:16.679] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:16.918+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:26:16.918] clusterConfig unchanged [2021-10-22T02:26:17.679+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:17.679] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:18.566+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:26:18.566] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:26:18.568+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:18.568] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:26:18.568+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [02:26:18.568] Could not get version : [02:26:18.568] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:26:18.568+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [02:26:18.568] Error getting version for log lines. Skipping : [02:26:18.568] Could not get version : [02:26:18.568] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:26:18.679+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:18.679] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:19.680+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:19.680] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:20.681+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:20.681] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:21.682+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:21.682] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:22.682+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:22.682] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:23.682+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:23.682] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:24.684+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:24.684] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:25.684+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:25.684] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:26.685+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:26.685] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:27.070+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:26:27.070] clusterConfig unchanged [2021-10-22T02:26:27.686+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:27.686] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:28.686+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:28.686] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:29.686+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:29.686] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:30.689+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:30.689] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:31.687+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:31.687] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:32.687+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:32.687] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:33.687+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:33.687] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:34.697+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:34.697] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:35.689+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:35.689] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:36.689+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:36.689] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:37.172+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:26:37.172] clusterConfig unchanged [2021-10-22T02:26:37.689+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:37.689] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:38.690+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:38.690] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:39.691+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:39.691] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:40.691+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:40.691] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:41.405+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:26:41.405] memory:rss=56709120 alloc=12524984 totalalloc=10840417992 mallocs=138364381 frees=138259713 heapinuse=17719296 heapobjects=104668 gcpausetotalns=1639386802 gcpauselastns=102695 gcnum=1589 utimens=235451027000 stimens=93462393000 minflt=582829 majflt=405 [2021-10-22T02:26:41.692+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:41.692] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:42.693+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:42.693] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:43.702+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:43.702] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:44.434+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [02:26:44.434] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:60760 (local=false) because it is not up [2021-10-22T02:26:44.694+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:44.694] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:45.695+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:45.695] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:46.207+0000] [.info] [src/director/director.go:computePlan:280] [02:26:46.207] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:26:46.207+0000] [.info] [src/director/director.go:tracef:794] [02:26:46.207] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:26:46.207+0000] [.info] [src/director/director.go:tracef:794] [02:26:46.207] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:26:46.207+0000] [.info] [src/director/director.go:tracef:794] [02:26:46.207] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:26:46.207+0000] [.info] [src/director/director.go:tracef:794] [02:26:46.207] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:26:46.208+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:26:46.208] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:26:46.378+0000] [.info] [src/director/director.go:tracef:794] [02:26:46.378] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:26:46.379+0000] [.info] [src/director/director.go:tracef:794] [02:26:46.378] because [2021-10-22T02:26:46.379+0000] [.info] [src/director/director.go:tracef:794] [02:26:46.379] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:26:46.379+0000] [.info] [src/director/director.go:tracef:794] [02:26:46.379] because ['currentState.CanBounceStop' = true] [2021-10-22T02:26:46.550+0000] [.info] [src/director/director.go:tracef:794] [02:26:46.550] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:26:46.550+0000] [.info] [src/director/director.go:tracef:794] [02:26:46.550] because [2021-10-22T02:26:46.551+0000] [.info] [src/director/director.go:tracef:794] [02:26:46.551] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:26:46.551+0000] [.info] [src/director/director.go:tracef:794] [02:26:46.551] because ['currentState.Up' = false] [2021-10-22T02:26:46.551+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:26:46.551] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:26:46.551+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:26:46.551] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:26:46.696+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:46.696] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:46.704+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:26:46.704] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:26:46.712+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:26:46.712] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:26:46.713+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:26:46.713] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:26:46.714+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:26:46.714] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:26:46.715+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:26:46.715] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:26:46.715+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:26:46.715] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:26:46.716+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:26:46.716] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:26:46.949+0000] [.info] [src/director/director.go:tracef:794] [02:26:46.949] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:26:46.949+0000] [.info] [src/director/director.go:tracef:794] [02:26:46.949] because ['currentState.Up' = false] [2021-10-22T02:26:47.161+0000] [.info] [src/director/director.go:tracef:794] [02:26:47.161] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:26:47.161+0000] [.info] [src/director/director.go:tracef:794] [02:26:47.161] because ['currentState.Up' = false] [2021-10-22T02:26:47.161+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:26:47.161] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:26:47.374+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:26:47.374] clusterConfig unchanged [2021-10-22T02:26:47.386+0000] [.info] [src/director/director.go:tracef:794] [02:26:47.386] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:26:47.386+0000] [.info] [src/director/director.go:tracef:794] [02:26:47.386] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:26:47.598+0000] [.info] [src/director/director.go:tracef:794] [02:26:47.598] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:26:47.598+0000] [.info] [src/director/director.go:tracef:794] [02:26:47.598] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:26:47.598+0000] [.info] [src/director/director.go:tracef:794] [02:26:47.598] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:26:47.598+0000] [.info] [src/director/director.go:tracef:794] [02:26:47.598] because ['currentState.Up' = false] [2021-10-22T02:26:47.598+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:26:47.598] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:26:47.598+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:26:47.598] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:26:47.629+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:26:47.629] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:26:47.630+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:26:47.630] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51767 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:26:47.728+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:47.728] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:48.354+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:26:48.354] ...Finished start [pid=18002] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51767,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:26:48.494+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:26:48.494] clusterConfig unchanged [2021-10-22T02:26:48.509+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:26:48.509] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 153.989979ms. [2021-10-22T02:26:48.509+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:26:48.509] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51767 (local=false) [2021-10-22T02:26:48.509+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:26:48.509] Dropping existing documents from system.replset [2021-10-22T02:26:48.626+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:26:48.626] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:26:48.630+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:26:48.630] Inserting oplogTruncateAfterPoint document [2021-10-22T02:26:48.630+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:26:48.630] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:26:48.631+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:26:48.631] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:26:48.631+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:26:48.631] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:26:48.654+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:26:48.654] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:26:48.654+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:26:48.654] Setting the minvalid document to [{_id ObjectID("61722168523370002becc87b")} {t -1} {ts {0 1}}] [2021-10-22T02:26:48.666+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:26:48.666] Inserted [{_id ObjectID("61722168523370002becc87b")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:26:48.666+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:26:48.666] Inserting new replica set configuration document. [2021-10-22T02:26:48.666+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:26:48.666] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:26:48.678+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:26:48.678] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:26:48.682+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:26:48.682] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:26:48.682+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:26:48.682] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:26:48.682+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:26:48.682] Initiating stopEphemeralMongo [2021-10-22T02:26:48.682+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:26:48.682] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51767 (local=false) [2021-10-22T02:26:48.682+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:26:48.682] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51767 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:26:48.704+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:48.704] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:49.046+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:26:49.046] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.499614ms. [2021-10-22T02:26:49.046+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:26:49.046] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51767 (local=false) [2021-10-22T02:26:49.046+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:26:49.046] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.541µs. [2021-10-22T02:26:49.046+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:26:49.046] Successfully finished running stopEphemeralMongo [2021-10-22T02:26:49.046+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:26:49.046] Successfully stopped existing ephemeral mongod. [2021-10-22T02:26:49.046+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:26:49.046] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:26:49.046+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:26:49.046] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:26:49.078+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:26:49.078] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:26:49.079+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:26:49.079] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35354 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:26:49.618+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:26:49.618] clusterConfig unchanged [2021-10-22T02:26:49.705+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:49.705] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:50.315+0000] [.error] [src/action/start.go:func1:93] [103] [02:26:50.315] Error starting mongod : [02:26:50.315] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35354,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35354 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:26:49.110Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 19416 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:26:49.142+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:26:49.142+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:26:49.142+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35354,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:26:49.144+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:26:49.146+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:26:49.147+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:26:49.161+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:26:49.161+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:26:49.163+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:26:49.210+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:26:49.210+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:26:49.765+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869609:765830][19416:0x7fa5c5b87bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:26:49.882+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869609:882776][19416:0x7fa5c5b87bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:26:49.994+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869609:994872][19416:0x7fa5c5b87bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:26:50.115+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869610:115188][19416:0x7fa5c5b87bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:26:50.190+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869610:190520][19416:0x7fa5c5b87bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:26:50.261+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869610:261584][19416:0x7fa5c5b87bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:26:50.261+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869610:261702][19416:0x7fa5c5b87bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:26:50.315+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:26:50.315] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35354 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:26:50.315] Error starting mongod : [02:26:50.315] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35354,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35354 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:26:49.110Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 19416 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:26:49.142+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:26:49.142+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:26:49.142+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35354,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:26:49.144+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:26:49.146+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:26:49.147+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:26:49.161+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:26:49.161+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:26:49.163+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:26:49.210+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:26:49.210+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:26:49.765+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869609:765830][19416:0x7fa5c5b87bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:26:49.882+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:26:50.315+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:26:50.315] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:26:50.315] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35354 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:26:50.315] Error starting mongod : [02:26:50.315] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35354,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35354 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:26:49.110Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 19416 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:26:49.142+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:26:49.142+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:26:49.142+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35354,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:26:49.144+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:26:49.146+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:26:49.147+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:26:49.161+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:26:49.161+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:26:49.163+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:26:49.210+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:26:49.210+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:26:49.765+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869609:765830][19416:0x7fa5c5b87bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:26:50.315+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:26:50.315] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:26:50.315+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:26:50.315] Initiating stopEphemeralMongo [2021-10-22T02:26:50.315+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:26:50.315] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:26:50.316+0000] [.error] [src/director/director.go:executePlan:962] [02:26:50.316] Failed to apply action. Result = : [02:26:50.315] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:26:50.315] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35354 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:26:50.315] Error starting mongod : [02:26:50.315] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35354,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35354 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:26:49.110Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 19416 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:26:49.142+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:26:49.142+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:26:49.142+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35354,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:26:49.144+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:26:49.146+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:26:49.147+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:26:49.161+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:26:49.161+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:26:49.163+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:26:49.210+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:26:49.210+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:26:49.765+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:26:50.316+0000] [.error] [src/director/director.go:planAndExecute:567] [02:26:50.316] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:26:50.316] Failed to apply action. Result = : [02:26:50.315] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:26:50.315] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35354 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:26:50.315] Error starting mongod : [02:26:50.315] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35354,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35354 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:26:49.110Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 19416 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:26:49.142+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:26:49.142+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:26:49.142+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35354,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:26:49.144+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:26:49.146+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:26:49.147+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:26:49.161+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:26:49.161+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:26:49.163+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:26:49.210+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:26:49.210+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:26:50.316+0000] [.error] [src/director/director.go:mainLoop:399] [02:26:50.316] Failed to planAndExecute : [02:26:50.316] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:26:50.316] Failed to apply action. Result = : [02:26:50.315] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:26:50.315] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35354 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:26:50.315] Error starting mongod : [02:26:50.315] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35354,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35354 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:26:49.110Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 19416 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:26:49.142+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:26:49.142+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:26:49.142+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35354,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:26:49.144+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:26:49.146+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:26:49.147+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:26:49.161+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:26:49.161+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:26:49.163+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:26:49.210+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:26:49.210+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:26:50.707+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:50.707] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:50.879+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:26:50.879] clusterConfig unchanged [2021-10-22T02:26:51.716+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:51.716] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:52.709+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:52.709] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:53.709+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:53.709] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:54.707+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:54.707] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:55.708+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:55.708] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:56.710+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:56.710] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:57.708+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:57.708] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:58.712+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:58.712] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:26:59.491+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:26:59.490] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.774655844s [2021-10-22T02:26:59.709+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:26:59.709] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:00.710+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:00.710] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:00.991+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:27:00.991] clusterConfig unchanged [2021-10-22T02:27:01.711+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:01.711] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:02.713+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:02.713] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:03.711+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:03.711] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:04.712+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:04.712] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:05.713+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:05.713] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:06.713+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:06.713] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:07.713+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:07.713] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:08.714+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:08.714] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:09.715+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:09.715] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:10.716+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:10.716] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:11.113+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:27:11.113] clusterConfig unchanged [2021-10-22T02:27:11.717+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:11.717] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:12.718+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:12.718] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:13.726+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:13.726] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:14.719+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:14.719] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:15.719+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:15.719] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:16.719+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:16.719] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:17.720+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:17.720] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:18.721+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:18.721] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:19.721+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:19.721] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:20.722+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:20.722] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:21.247+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:27:21.247] clusterConfig unchanged [2021-10-22T02:27:21.722+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:21.722] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:22.723+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:22.723] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:23.723+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:23.723] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:24.724+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:24.724] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:25.732+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:25.732] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:26.725+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:26.725] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:27.571+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:27:27.571] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:27:27.572+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:27:27.572] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:27:27.726+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:27.726] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:28.726+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:28.726] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:29.727+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:29.727] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:30.727+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:30.727] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:31.376+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:27:31.376] clusterConfig unchanged [2021-10-22T02:27:31.731+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:31.731] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:32.728+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:32.728] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:33.729+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:33.729] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:34.729+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:34.729] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:35.729+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:35.729] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:36.730+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:36.730] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:37.739+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:37.739] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:38.733+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [02:27:38.733] lsblk cache invalidated, running lsblk command [2021-10-22T02:27:38.734+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:38.734] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:39.732+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:39.732] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:40.732+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:40.732] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:41.406+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:27:41.406] memory:rss=58400768 alloc=10755424 totalalloc=10934214184 mallocs=139566507 frees=139482658 heapinuse=17530880 heapobjects=83849 gcpausetotalns=1641900563 gcpauselastns=113333 gcnum=1601 utimens=237868128000 stimens=94515723000 minflt=587983 majflt=405 [2021-10-22T02:27:41.488+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:27:41.488] clusterConfig unchanged [2021-10-22T02:27:41.733+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:41.733] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:42.736+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:42.736] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:43.734+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:43.734] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:44.735+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:44.735] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:45.736+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:45.736] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:46.736+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:46.736] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:47.736+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:47.736] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:48.737+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:48.736] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:49.737+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:49.737] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:50.490+0000] [.info] [src/director/director.go:computePlan:280] [02:27:50.490] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:27:50.491+0000] [.info] [src/director/director.go:tracef:794] [02:27:50.491] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:27:50.491+0000] [.info] [src/director/director.go:tracef:794] [02:27:50.491] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:27:50.491+0000] [.info] [src/director/director.go:tracef:794] [02:27:50.491] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:27:50.491+0000] [.info] [src/director/director.go:tracef:794] [02:27:50.491] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:27:50.491+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:27:50.491] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:27:50.639+0000] [.info] [src/director/director.go:tracef:794] [02:27:50.639] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:27:50.639+0000] [.info] [src/director/director.go:tracef:794] [02:27:50.639] because [2021-10-22T02:27:50.639+0000] [.info] [src/director/director.go:tracef:794] [02:27:50.639] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:27:50.639+0000] [.info] [src/director/director.go:tracef:794] [02:27:50.639] because ['currentState.CanBounceStop' = true] [2021-10-22T02:27:50.740+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:50.740] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:50.805+0000] [.info] [src/director/director.go:tracef:794] [02:27:50.805] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:27:50.805+0000] [.info] [src/director/director.go:tracef:794] [02:27:50.805] because [2021-10-22T02:27:50.805+0000] [.info] [src/director/director.go:tracef:794] [02:27:50.805] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:27:50.805+0000] [.info] [src/director/director.go:tracef:794] [02:27:50.805] because ['currentState.Up' = false] [2021-10-22T02:27:50.805+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:27:50.805] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:27:50.805+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:27:50.805] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:27:50.901+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:27:50.901] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:27:50.909+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:27:50.909] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:27:50.910+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:27:50.910] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:27:50.911+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:27:50.911] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:27:50.912+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:27:50.912] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:27:50.912+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:27:50.912] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:27:50.912+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:27:50.912] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:27:50.913+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:27:50.913] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:27:51.128+0000] [.info] [src/director/director.go:tracef:794] [02:27:51.128] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:27:51.128+0000] [.info] [src/director/director.go:tracef:794] [02:27:51.128] because ['currentState.Up' = false] [2021-10-22T02:27:51.338+0000] [.info] [src/director/director.go:tracef:794] [02:27:51.338] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:27:51.338+0000] [.info] [src/director/director.go:tracef:794] [02:27:51.338] because ['currentState.Up' = false] [2021-10-22T02:27:51.339+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:27:51.339] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:27:51.567+0000] [.info] [src/director/director.go:tracef:794] [02:27:51.567] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:27:51.567+0000] [.info] [src/director/director.go:tracef:794] [02:27:51.567] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:27:51.722+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:27:51.722] clusterConfig unchanged [2021-10-22T02:27:51.740+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:51.740] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:51.793+0000] [.info] [src/director/director.go:tracef:794] [02:27:51.793] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:27:51.793+0000] [.info] [src/director/director.go:tracef:794] [02:27:51.793] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:27:51.793+0000] [.info] [src/director/director.go:tracef:794] [02:27:51.793] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:27:51.793+0000] [.info] [src/director/director.go:tracef:794] [02:27:51.793] because ['currentState.Up' = false] [2021-10-22T02:27:51.793+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:27:51.793] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:27:51.793+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:27:51.793] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:27:51.827+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:27:51.827] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:27:51.828+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:27:51.828] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59210 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:27:52.560+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:27:52.560] ...Finished start [pid=38673] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59210,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:27:52.704+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:27:52.704] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 143.142664ms. [2021-10-22T02:27:52.704+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:27:52.704] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:59210 (local=false) [2021-10-22T02:27:52.704+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:27:52.704] Dropping existing documents from system.replset [2021-10-22T02:27:52.751+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:52.751] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:52.831+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:27:52.831] clusterConfig unchanged [2021-10-22T02:27:52.847+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:27:52.847] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:27:52.851+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:27:52.851] Inserting oplogTruncateAfterPoint document [2021-10-22T02:27:52.851+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:27:52.851] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:27:52.852+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:27:52.852] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:27:52.852+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:27:52.852] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:27:52.864+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:27:52.864] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:27:52.864+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:27:52.864] Setting the minvalid document to [{_id ObjectID("617221a8523370002becc889")} {t -1} {ts {0 1}}] [2021-10-22T02:27:52.877+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:27:52.877] Inserted [{_id ObjectID("617221a8523370002becc889")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:27:52.878+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:27:52.878] Inserting new replica set configuration document. [2021-10-22T02:27:52.878+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:27:52.878] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:27:52.891+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:27:52.890] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:27:52.894+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:27:52.894] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:27:52.894+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:27:52.894] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:27:52.894+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:27:52.894] Initiating stopEphemeralMongo [2021-10-22T02:27:52.894+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:27:52.894] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:59210 (local=false) [2021-10-22T02:27:52.894+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:27:52.894] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:59210 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:27:53.283+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:27:53.283] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.263217ms. [2021-10-22T02:27:53.283+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:27:53.283] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:59210 (local=false) [2021-10-22T02:27:53.283+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:27:53.283] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 40.186µs. [2021-10-22T02:27:53.283+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:27:53.283] Successfully finished running stopEphemeralMongo [2021-10-22T02:27:53.283+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:27:53.283] Successfully stopped existing ephemeral mongod. [2021-10-22T02:27:53.283+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:27:53.283] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:27:53.283+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:27:53.283] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:27:53.317+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:27:53.317] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:27:53.318+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:27:53.318] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38950 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:27:53.742+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:53.742] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:53.963+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:27:53.963] clusterConfig unchanged [2021-10-22T02:27:54.569+0000] [.error] [src/action/start.go:func1:93] [103] [02:27:54.569] Error starting mongod : [02:27:54.569] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38950,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38950 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:27:53.350Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 40178 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:27:53.370+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:27:53.370+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:27:53.370+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38950,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:27:53.373+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:27:53.374+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:27:53.375+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:27:53.444+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:27:53.444+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:27:53.446+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:27:53.496+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:27:53.496+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:27:54.030+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869674:30073][40178:0x7f259403dbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:27:54.147+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869674:147703][40178:0x7f259403dbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:27:54.251+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869674:251610][40178:0x7f259403dbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:27:54.368+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869674:368423][40178:0x7f259403dbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:27:54.444+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869674:444740][40178:0x7f259403dbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:27:54.508+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869674:508633][40178:0x7f259403dbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:27:54.508+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869674:508708][40178:0x7f259403dbc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:27:54.569+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:27:54.569] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38950 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:27:54.569] Error starting mongod : [02:27:54.569] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38950,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38950 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:27:53.350Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 40178 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:27:53.370+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:27:53.370+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:27:53.370+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38950,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:27:53.373+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:27:53.374+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:27:53.375+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:27:53.444+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:27:53.444+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:27:53.446+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:27:53.496+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:27:53.496+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:27:54.030+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869674:30073][40178:0x7f259403dbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:27:54.147+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:27:54.570+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:27:54.569] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:27:54.569] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38950 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:27:54.569] Error starting mongod : [02:27:54.569] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38950,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38950 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:27:53.350Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 40178 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:27:53.370+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:27:53.370+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:27:53.370+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38950,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:27:53.373+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:27:53.374+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:27:53.375+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:27:53.444+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:27:53.444+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:27:53.446+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:27:53.496+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:27:53.496+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:27:54.030+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869674:30073][40178:0x7f259403dbc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:27:54.570+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:27:54.570] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:27:54.570+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:27:54.570] Initiating stopEphemeralMongo [2021-10-22T02:27:54.570+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:27:54.570] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:27:54.570+0000] [.error] [src/director/director.go:executePlan:962] [02:27:54.570] Failed to apply action. Result = : [02:27:54.569] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:27:54.569] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38950 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:27:54.569] Error starting mongod : [02:27:54.569] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38950,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38950 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:27:53.350Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 40178 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:27:53.370+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:27:53.370+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:27:53.370+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38950,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:27:53.373+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:27:53.374+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:27:53.375+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:27:53.444+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:27:53.444+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:27:53.446+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:27:53.496+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:27:53.496+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:27:54.030+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:27:54.570+0000] [.error] [src/director/director.go:planAndExecute:567] [02:27:54.570] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:27:54.570] Failed to apply action. Result = : [02:27:54.569] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:27:54.569] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38950 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:27:54.569] Error starting mongod : [02:27:54.569] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38950,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38950 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:27:53.350Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 40178 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:27:53.370+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:27:53.370+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:27:53.370+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38950,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:27:53.373+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:27:53.374+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:27:53.375+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:27:53.444+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:27:53.444+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:27:53.446+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:27:53.496+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:27:53.496+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:27:54.570+0000] [.error] [src/director/director.go:mainLoop:399] [02:27:54.570] Failed to planAndExecute : [02:27:54.570] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:27:54.570] Failed to apply action. Result = : [02:27:54.569] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:27:54.569] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:38950 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:27:54.569] Error starting mongod : [02:27:54.569] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38950,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38950 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:27:53.350Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 40178 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:27:53.370+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:27:53.370+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:27:53.370+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38950,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:27:53.373+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:27:53.374+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:27:53.375+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:27:53.444+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:27:53.444+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:27:53.446+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:27:53.496+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:27:53.496+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:27:54.743+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:54.743] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:55.224+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:27:55.224] clusterConfig unchanged [2021-10-22T02:27:55.743+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:55.743] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:56.742+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:56.742] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:57.743+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:57.743] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:58.743+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:58.742] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:27:59.744+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:27:59.744] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:00.744+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:00.744] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:01.754+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:01.754] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:02.753+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:02.753] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:03.755+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:03.755] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:03.783+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:28:03.783] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.869949939s [2021-10-22T02:28:04.752+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:04.752] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:05.355+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:28:05.355] clusterConfig unchanged [2021-10-22T02:28:05.753+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:05.753] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:06.753+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:06.753] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:07.753+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:07.753] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:08.754+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:08.754] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:09.756+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:09.756] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:10.755+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:10.755] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:11.756+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:11.756] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:12.756+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:12.756] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:13.757+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:13.757] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:14.757+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:14.757] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:15.477+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:28:15.477] clusterConfig unchanged [2021-10-22T02:28:15.758+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:15.758] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:16.758+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:16.758] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:17.759+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:17.759] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:18.760+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:18.760] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:19.761+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:19.761] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:20.769+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:20.769] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:21.767+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:21.767] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:22.765+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:22.765] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:23.762+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:23.762] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:24.764+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:24.764] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:25.609+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:28:25.609] clusterConfig unchanged [2021-10-22T02:28:25.764+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:25.764] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:26.764+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:26.764] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:27.764+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:27.764] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:28.765+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:28.765] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:29.765+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:29.765] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:30.765+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:30.765] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:31.767+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:31.767] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:32.767+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:32.767] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:33.768+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:33.767] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:34.768+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:34.768] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:35.730+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:28:35.730] clusterConfig unchanged [2021-10-22T02:28:35.768+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:35.768] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:36.770+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:36.770] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:37.771+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:37.771] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:38.773+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:38.773] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:39.772+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:39.772] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:40.772+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:40.772] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:41.407+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:28:41.407] memory:rss=57102336 alloc=14046784 totalalloc=11027245872 mallocs=140766039 frees=140644021 heapinuse=18325504 heapobjects=122018 gcpausetotalns=1644199389 gcpauselastns=69034 gcnum=1612 utimens=240406204000 stimens=95683188000 minflt=593235 majflt=405 [2021-10-22T02:28:41.772+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:41.772] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:42.772+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:42.772] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:43.773+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:43.773] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:44.774+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:44.774] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:45.774+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:45.774] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:45.863+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:28:45.863] clusterConfig unchanged [2021-10-22T02:28:46.777+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:46.777] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:47.775+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:47.775] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:48.776+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:48.776] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:49.776+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:49.776] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:50.777+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:50.777] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:51.777+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:51.777] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:52.778+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:52.778] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:53.778+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:53.778] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:54.737+0000] [.info] [src/director/director.go:computePlan:280] [02:28:54.737] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:28:54.738+0000] [.info] [src/director/director.go:tracef:794] [02:28:54.738] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:28:54.738+0000] [.info] [src/director/director.go:tracef:794] [02:28:54.738] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:28:54.738+0000] [.info] [src/director/director.go:tracef:794] [02:28:54.738] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:28:54.738+0000] [.info] [src/director/director.go:tracef:794] [02:28:54.738] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:28:54.738+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:28:54.738] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:28:54.784+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:54.784] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:54.894+0000] [.info] [src/director/director.go:tracef:794] [02:28:54.894] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:28:54.894+0000] [.info] [src/director/director.go:tracef:794] [02:28:54.894] because [2021-10-22T02:28:54.894+0000] [.info] [src/director/director.go:tracef:794] [02:28:54.894] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:28:54.894+0000] [.info] [src/director/director.go:tracef:794] [02:28:54.894] because ['currentState.CanBounceStop' = true] [2021-10-22T02:28:55.050+0000] [.info] [src/director/director.go:tracef:794] [02:28:55.050] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:28:55.050+0000] [.info] [src/director/director.go:tracef:794] [02:28:55.050] because [2021-10-22T02:28:55.050+0000] [.info] [src/director/director.go:tracef:794] [02:28:55.050] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:28:55.050+0000] [.info] [src/director/director.go:tracef:794] [02:28:55.050] because ['currentState.Up' = false] [2021-10-22T02:28:55.050+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:28:55.050] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:28:55.050+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:28:55.050] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:28:55.137+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:28:55.137] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:28:55.138+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:28:55.138] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:28:55.145+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:28:55.145] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:28:55.147+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:28:55.147] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:28:55.148+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:28:55.148] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:28:55.148+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:28:55.148] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:28:55.148+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:28:55.148] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:28:55.149+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:28:55.149] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:28:55.354+0000] [.info] [src/director/director.go:tracef:794] [02:28:55.354] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:28:55.354+0000] [.info] [src/director/director.go:tracef:794] [02:28:55.354] because ['currentState.Up' = false] [2021-10-22T02:28:55.570+0000] [.info] [src/director/director.go:tracef:794] [02:28:55.570] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:28:55.570+0000] [.info] [src/director/director.go:tracef:794] [02:28:55.570] because ['currentState.Up' = false] [2021-10-22T02:28:55.570+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:28:55.570] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:28:55.771+0000] [.info] [src/director/director.go:tracef:794] [02:28:55.771] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:28:55.771+0000] [.info] [src/director/director.go:tracef:794] [02:28:55.771] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:28:55.789+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:55.789] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:55.978+0000] [.info] [src/director/director.go:tracef:794] [02:28:55.978] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:28:55.978+0000] [.info] [src/director/director.go:tracef:794] [02:28:55.978] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:28:55.978+0000] [.info] [src/director/director.go:tracef:794] [02:28:55.978] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:28:55.978+0000] [.info] [src/director/director.go:tracef:794] [02:28:55.978] because ['currentState.Up' = false] [2021-10-22T02:28:55.978+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:28:55.978] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:28:55.978+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:28:55.978] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:28:56.008+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:28:56.008] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:28:56.008+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:28:56.008] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 33135 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:28:56.088+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:28:56.088] clusterConfig unchanged [2021-10-22T02:28:56.747+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:28:56.747] ...Finished start [pid=59453] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":33135,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:28:56.782+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:56.782] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:56.865+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:28:56.865] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 117.244688ms. [2021-10-22T02:28:56.865+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:28:56.865] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:33135 (local=false) [2021-10-22T02:28:56.865+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:28:56.865] Dropping existing documents from system.replset [2021-10-22T02:28:56.993+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:28:56.993] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:28:56.997+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:28:56.997] Inserting oplogTruncateAfterPoint document [2021-10-22T02:28:56.997+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:28:56.997] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:28:56.998+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:28:56.998] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:28:56.998+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:28:56.998] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:28:57.014+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:28:57.014] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:28:57.014+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:28:57.014] Setting the minvalid document to [{_id ObjectID("617221e9523370002becc897")} {t -1} {ts {0 1}}] [2021-10-22T02:28:57.027+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:28:57.027] Inserted [{_id ObjectID("617221e9523370002becc897")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:28:57.027+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:28:57.027] Inserting new replica set configuration document. [2021-10-22T02:28:57.027+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:28:57.027] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:28:57.041+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:28:57.041] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:28:57.057+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:28:57.057] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:28:57.057+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:28:57.057] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:28:57.057+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:28:57.057] Initiating stopEphemeralMongo [2021-10-22T02:28:57.057+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:28:57.057] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:33135 (local=false) [2021-10-22T02:28:57.057+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:28:57.057] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:33135 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:28:57.215+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:28:57.215] clusterConfig unchanged [2021-10-22T02:28:57.473+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:28:57.473] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.143817ms. [2021-10-22T02:28:57.473+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:28:57.473] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:33135 (local=false) [2021-10-22T02:28:57.474+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:28:57.474] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.616µs. [2021-10-22T02:28:57.474+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:28:57.474] Successfully finished running stopEphemeralMongo [2021-10-22T02:28:57.474+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:28:57.474] Successfully stopped existing ephemeral mongod. [2021-10-22T02:28:57.474+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:28:57.474] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:28:57.474+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:28:57.474] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:28:57.510+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:28:57.510] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:28:57.511+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:28:57.511] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37093 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:28:57.784+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:57.784] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:58.321+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:28:58.321] clusterConfig unchanged [2021-10-22T02:28:58.664+0000] [.error] [src/action/start.go:func1:93] [103] [02:28:58.664] Error starting mongod : [02:28:58.664] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37093,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37093 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:28:57.544Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 61005 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:28:57.565+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:28:57.565+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:28:57.565+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37093,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:28:57.566+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:28:57.568+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:28:57.568+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:28:57.608+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:28:57.608+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:28:57.609+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:28:57.671+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:28:57.671+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:28:58.167+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869738:167274][61005:0x7faf4bcc4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:28:58.278+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869738:278312][61005:0x7faf4bcc4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:28:58.377+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869738:377545][61005:0x7faf4bcc4bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:28:58.484+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869738:483985][61005:0x7faf4bcc4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:28:58.554+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869738:554843][61005:0x7faf4bcc4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:28:58.615+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869738:615180][61005:0x7faf4bcc4bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:28:58.615+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869738:615254][61005:0x7faf4bcc4bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:28:58.664+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:28:58.664] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37093 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:28:58.664] Error starting mongod : [02:28:58.664] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37093,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37093 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:28:57.544Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 61005 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:28:57.565+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:28:57.565+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:28:57.565+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37093,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:28:57.566+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:28:57.568+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:28:57.568+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:28:57.608+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:28:57.608+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:28:57.609+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:28:57.671+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:28:57.671+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:28:58.167+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869738:167274][61005:0x7faf4bcc4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:28:58.278+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:28:58.664+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:28:58.664] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:28:58.664] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37093 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:28:58.664] Error starting mongod : [02:28:58.664] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37093,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37093 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:28:57.544Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 61005 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:28:57.565+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:28:57.565+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:28:57.565+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37093,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:28:57.566+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:28:57.568+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:28:57.568+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:28:57.608+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:28:57.608+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:28:57.609+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:28:57.671+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:28:57.671+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:28:58.167+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869738:167274][61005:0x7faf4bcc4bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:28:58.665+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:28:58.665] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:28:58.665+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:28:58.665] Initiating stopEphemeralMongo [2021-10-22T02:28:58.665+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:28:58.665] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:28:58.665+0000] [.error] [src/director/director.go:executePlan:962] [02:28:58.665] Failed to apply action. Result = : [02:28:58.664] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:28:58.664] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37093 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:28:58.664] Error starting mongod : [02:28:58.664] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37093,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37093 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:28:57.544Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 61005 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:28:57.565+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:28:57.565+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:28:57.565+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37093,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:28:57.566+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:28:57.568+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:28:57.568+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:28:57.608+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:28:57.608+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:28:57.609+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:28:57.671+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:28:57.671+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:28:58.167+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:28:58.665+0000] [.error] [src/director/director.go:planAndExecute:567] [02:28:58.665] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:28:58.665] Failed to apply action. Result = : [02:28:58.664] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:28:58.664] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37093 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:28:58.664] Error starting mongod : [02:28:58.664] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37093,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37093 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:28:57.544Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 61005 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:28:57.565+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:28:57.565+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:28:57.565+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37093,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:28:57.566+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:28:57.568+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:28:57.568+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:28:57.608+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:28:57.608+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:28:57.609+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:28:57.671+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:28:57.671+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:28:58.665+0000] [.error] [src/director/director.go:mainLoop:399] [02:28:58.665] Failed to planAndExecute : [02:28:58.665] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:28:58.665] Failed to apply action. Result = : [02:28:58.664] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:28:58.664] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37093 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:28:58.664] Error starting mongod : [02:28:58.664] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37093,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37093 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:28:57.544Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 61005 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:28:57.565+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:28:57.565+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:28:57.565+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37093,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:28:57.566+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:28:57.568+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:28:57.568+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:28:57.608+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:28:57.608+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:28:57.609+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:28:57.671+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:28:57.671+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:28:58.784+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:58.784] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:28:59.578+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:28:59.578] clusterConfig unchanged [2021-10-22T02:28:59.783+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:28:59.783] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:00.792+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:00.792] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:01.801+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:01.801] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:02.800+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:02.800] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:03.803+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:03.803] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:04.803+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:04.803] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:05.821+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:05.821] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:06.804+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:06.804] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:07.802+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:07.802] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:07.866+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:29:07.866] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.716579832s [2021-10-22T02:29:08.801+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:08.801] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:09.680+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:29:09.680] clusterConfig unchanged [2021-10-22T02:29:09.803+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:09.803] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:10.804+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:10.804] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:11.804+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:11.804] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:12.805+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:12.805] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:13.805+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:13.805] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:14.805+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:14.805] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:15.806+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:15.806] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:16.808+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:16.808] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:17.815+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:17.815] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:18.808+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:18.808] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:19.783+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:29:19.783] clusterConfig unchanged [2021-10-22T02:29:19.808+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:19.808] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:20.808+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:20.808] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:21.809+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:21.809] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:22.810+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:22.809] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:23.810+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:23.810] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:24.811+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:24.811] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:25.812+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:25.812] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:26.812+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:26.812] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:27.813+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:27.813] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:28.813+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:28.813] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:29.820+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:29.820] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:29.914+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:29:29.914] clusterConfig unchanged [2021-10-22T02:29:30.816+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:30.816] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:31.815+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:31.815] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:32.816+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:32.815] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:33.816+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:33.816] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:34.817+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:34.817] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:35.819+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:35.819] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:36.819+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:36.819] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:37.819+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:37.819] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:38.820+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:38.820] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:39.821+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:39.821] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:40.075+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:29:40.075] clusterConfig unchanged [2021-10-22T02:29:40.821+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:40.821] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:41.407+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:29:41.407] memory:rss=58863616 alloc=15981616 totalalloc=11119348784 mallocs=141953465 frees=141807523 heapinuse=19095552 heapobjects=145942 gcpausetotalns=1646260472 gcpauselastns=77337 gcnum=1623 utimens=242793672000 stimens=96781767000 minflt=599027 majflt=405 [2021-10-22T02:29:41.822+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:41.822] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:42.822+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:42.822] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:43.823+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:43.823] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:44.441+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [02:29:44.441] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37093 (local=false) because it is not up [2021-10-22T02:29:44.823+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:44.823] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:45.824+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:45.824] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:46.825+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:46.825] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:47.824+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:47.824] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:48.825+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:48.825] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:49.834+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:49.834] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:50.197+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:29:50.197] clusterConfig unchanged [2021-10-22T02:29:50.826+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:50.826] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:51.826+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:51.826] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:52.828+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:52.828] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:53.828+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:53.828] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:54.829+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:54.829] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:55.829+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:55.829] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:56.830+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:56.830] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:57.831+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:57.831] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:58.832+0000] [.info] [src/director/director.go:computePlan:280] [02:29:58.832] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:29:58.833+0000] [.info] [src/director/director.go:tracef:794] [02:29:58.833] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:29:58.833+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:58.833] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:58.834+0000] [.info] [src/director/director.go:tracef:794] [02:29:58.834] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:29:58.835+0000] [.info] [src/director/director.go:tracef:794] [02:29:58.835] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:29:58.835+0000] [.info] [src/director/director.go:tracef:794] [02:29:58.835] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:29:58.835+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:29:58.835] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:29:58.993+0000] [.info] [src/director/director.go:tracef:794] [02:29:58.993] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:29:58.993+0000] [.info] [src/director/director.go:tracef:794] [02:29:58.993] because [2021-10-22T02:29:58.993+0000] [.info] [src/director/director.go:tracef:794] [02:29:58.993] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:29:58.993+0000] [.info] [src/director/director.go:tracef:794] [02:29:58.993] because ['currentState.CanBounceStop' = true] [2021-10-22T02:29:59.174+0000] [.info] [src/director/director.go:tracef:794] [02:29:59.174] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:29:59.174+0000] [.info] [src/director/director.go:tracef:794] [02:29:59.174] because [2021-10-22T02:29:59.174+0000] [.info] [src/director/director.go:tracef:794] [02:29:59.174] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:29:59.174+0000] [.info] [src/director/director.go:tracef:794] [02:29:59.174] because ['currentState.Up' = false] [2021-10-22T02:29:59.175+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:29:59.174] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:29:59.175+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:29:59.175] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:29:59.256+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:29:59.256] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:29:59.264+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:29:59.264] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:29:59.264+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:29:59.264] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:29:59.265+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:29:59.265] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:29:59.266+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:29:59.266] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:29:59.266+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:29:59.266] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:29:59.266+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:29:59.266] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:29:59.268+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:29:59.268] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:29:59.471+0000] [.info] [src/director/director.go:tracef:794] [02:29:59.471] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:29:59.471+0000] [.info] [src/director/director.go:tracef:794] [02:29:59.471] because ['currentState.Up' = false] [2021-10-22T02:29:59.672+0000] [.info] [src/director/director.go:tracef:794] [02:29:59.672] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:29:59.672+0000] [.info] [src/director/director.go:tracef:794] [02:29:59.672] because ['currentState.Up' = false] [2021-10-22T02:29:59.672+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:29:59.672] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:29:59.835+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:29:59.835] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:29:59.882+0000] [.info] [src/director/director.go:tracef:794] [02:29:59.882] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:29:59.883+0000] [.info] [src/director/director.go:tracef:794] [02:29:59.882] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:30:00.095+0000] [.info] [src/director/director.go:tracef:794] [02:30:00.095] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:30:00.095+0000] [.info] [src/director/director.go:tracef:794] [02:30:00.095] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:30:00.095+0000] [.info] [src/director/director.go:tracef:794] [02:30:00.095] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:30:00.095+0000] [.info] [src/director/director.go:tracef:794] [02:30:00.095] because ['currentState.Up' = false] [2021-10-22T02:30:00.095+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:30:00.095] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:30:00.095+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:30:00.095] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:30:00.129+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:30:00.129] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:30:00.130+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:30:00.130] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49029 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:30:00.390+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:30:00.390] clusterConfig unchanged [2021-10-22T02:30:00.835+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:00.835] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:00.876+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:30:00.875] ...Finished start [pid=6155] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49029,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:30:01.017+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:30:01.017] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 141.259049ms. [2021-10-22T02:30:01.017+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:30:01.017] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49029 (local=false) [2021-10-22T02:30:01.017+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:30:01.017] Dropping existing documents from system.replset [2021-10-22T02:30:01.165+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:30:01.165] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:30:01.174+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:30:01.174] Inserting oplogTruncateAfterPoint document [2021-10-22T02:30:01.174+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:30:01.174] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:30:01.176+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:30:01.176] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:30:01.176+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:30:01.176] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:30:01.189+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:30:01.189] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:30:01.189+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:30:01.189] Setting the minvalid document to [{_id ObjectID("61722229523370002becc8a5")} {t -1} {ts {0 1}}] [2021-10-22T02:30:01.201+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:30:01.201] Inserted [{_id ObjectID("61722229523370002becc8a5")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:30:01.201+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:30:01.201] Inserting new replica set configuration document. [2021-10-22T02:30:01.201+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:30:01.201] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:30:01.215+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:30:01.215] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:30:01.259+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:30:01.259] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:30:01.259+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:30:01.259] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:30:01.259+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:30:01.259] Initiating stopEphemeralMongo [2021-10-22T02:30:01.259+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:30:01.259] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49029 (local=false) [2021-10-22T02:30:01.259+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:30:01.259] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49029 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:30:01.520+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:30:01.520] clusterConfig unchanged [2021-10-22T02:30:01.711+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:30:01.711] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.705165ms. [2021-10-22T02:30:01.711+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:30:01.711] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49029 (local=false) [2021-10-22T02:30:01.711+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:30:01.711] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 36.899µs. [2021-10-22T02:30:01.711+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:30:01.711] Successfully finished running stopEphemeralMongo [2021-10-22T02:30:01.711+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:30:01.711] Successfully stopped existing ephemeral mongod. [2021-10-22T02:30:01.711+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:30:01.711] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:30:01.711+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:30:01.711] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:30:01.767+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:30:01.767] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:30:01.768+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:30:01.768] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39302 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:30:01.837+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:01.837] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:02.726+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:30:02.726] clusterConfig unchanged [2021-10-22T02:30:02.841+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:02.841] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:03.038+0000] [.error] [src/action/start.go:func1:93] [103] [02:30:03.038] Error starting mongod : [02:30:03.038] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39302 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:30:01.809Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 7594 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:30:01.835+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:30:01.835+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:30:01.835+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:30:01.838+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:30:01.839+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:30:01.841+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:30:01.868+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:30:01.868+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:30:01.868+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:30:01.928+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:30:01.928+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:30:02.462+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869802:462434][7594:0x7f5126578bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:30:02.569+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869802:569455][7594:0x7f5126578bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:30:02.699+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869802:699828][7594:0x7f5126578bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T02:30:02.826+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869802:826038][7594:0x7f5126578bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:30:02.917+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869802:917698][7594:0x7f5126578bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:30:02.985+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869802:985542][7594:0x7f5126578bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:30:02.985+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869802:985604][7594:0x7f5126578bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:30:03.038+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:30:03.038] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39302 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:30:03.038] Error starting mongod : [02:30:03.038] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39302 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:30:01.809Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 7594 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:30:01.835+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:30:01.835+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:30:01.835+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:30:01.838+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:30:01.839+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:30:01.841+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:30:01.868+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:30:01.868+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:30:01.868+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:30:01.928+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:30:01.928+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:30:02.462+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869802:462434][7594:0x7f5126578bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:30:02.569+00:00"},"s...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:30:03.039+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:30:03.038] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:30:03.038] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39302 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:30:03.038] Error starting mongod : [02:30:03.038] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39302 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:30:01.809Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 7594 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:30:01.835+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:30:01.835+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:30:01.835+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:30:01.838+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:30:01.839+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:30:01.841+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:30:01.868+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:30:01.868+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:30:01.868+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:30:01.928+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:30:01.928+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:30:02.462+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869802:462434][7594:0x7f5126578bc0... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:30:03.039+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:30:03.039] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:30:03.039+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:30:03.039] Initiating stopEphemeralMongo [2021-10-22T02:30:03.039+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:30:03.039] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:30:03.039+0000] [.error] [src/director/director.go:executePlan:962] [02:30:03.039] Failed to apply action. Result = : [02:30:03.038] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:30:03.038] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39302 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:30:03.038] Error starting mongod : [02:30:03.038] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39302 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:30:01.809Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 7594 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:30:01.835+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:30:01.835+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:30:01.835+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:30:01.838+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:30:01.839+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:30:01.841+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:30:01.868+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:30:01.868+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:30:01.868+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:30:01.928+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:30:01.928+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:30:02.462+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:30:03.039+0000] [.error] [src/director/director.go:planAndExecute:567] [02:30:03.039] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:30:03.039] Failed to apply action. Result = : [02:30:03.038] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:30:03.038] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39302 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:30:03.038] Error starting mongod : [02:30:03.038] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39302 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:30:01.809Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 7594 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:30:01.835+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:30:01.835+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:30:01.835+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:30:01.838+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:30:01.839+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:30:01.841+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:30:01.868+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:30:01.868+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:30:01.868+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:30:01.928+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:30:01.928+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:30:03.039+0000] [.error] [src/director/director.go:mainLoop:399] [02:30:03.039] Failed to planAndExecute : [02:30:03.039] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:30:03.039] Failed to apply action. Result = : [02:30:03.038] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:30:03.038] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:39302 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:30:03.038] Error starting mongod : [02:30:03.038] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39302 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:30:01.809Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 7594 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: ig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:30:01.835+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:30:01.835+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:30:01.835+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39302,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:30:01.838+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:30:01.839+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:30:01.841+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:30:01.868+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:30:01.868+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:30:01.868+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:30:01.928+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:30:01.928+00:00"},"s":"I", "c":"NETWORK", "id"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:30:03.840+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:03.840] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:03.952+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:30:03.952] clusterConfig unchanged [2021-10-22T02:30:04.839+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:04.839] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:05.842+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:05.842] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:06.841+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:06.841] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:07.846+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:07.846] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:08.843+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:08.843] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:09.841+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:09.841] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:10.842+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:10.842] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:11.845+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:11.845] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:12.304+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:30:12.304] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 13.035934015s [2021-10-22T02:30:12.843+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:12.843] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:13.843+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:13.843] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:14.085+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:30:14.085] clusterConfig unchanged [2021-10-22T02:30:14.844+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:14.844] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:15.844+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:15.844] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:16.845+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:16.845] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:17.845+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:17.845] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:18.853+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:18.853] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:19.846+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:19.846] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:20.848+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:20.847] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:21.848+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:21.848] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:22.849+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:22.849] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:23.850+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:23.850] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:24.215+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:30:24.215] clusterConfig unchanged [2021-10-22T02:30:24.850+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:24.850] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:25.851+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:25.851] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:26.851+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:26.851] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:27.852+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:27.852] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:28.852+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:28.852] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:29.852+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:29.852] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:30.854+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:30.853] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:31.856+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:31.856] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:32.855+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:32.855] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:33.854+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:33.854] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:34.337+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:30:34.337] clusterConfig unchanged [2021-10-22T02:30:34.855+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:34.855] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:35.857+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:35.857] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:36.857+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:36.857] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:37.862+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:37.862] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:38.858+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:38.858] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:39.858+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:39.858] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:40.858+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:40.858] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:41.408+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:30:41.408] memory:rss=57716736 alloc=8284072 totalalloc=11211862728 mallocs=143150637 frees=143094711 heapinuse=16908288 heapobjects=55926 gcpausetotalns=1647277810 gcpauselastns=51339 gcnum=1636 utimens=245427432000 stimens=97961679000 minflt=604195 majflt=405 [2021-10-22T02:30:41.859+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:41.859] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:42.860+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:42.859] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:43.861+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:43.861] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:44.439+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:30:44.439] clusterConfig unchanged [2021-10-22T02:30:44.862+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:44.862] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:45.862+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:45.862] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:46.862+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:46.862] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:47.864+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:47.864] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:48.864+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:48.864] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:49.864+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:49.864] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:50.865+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:50.865] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:51.866+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:51.866] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:52.867+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:52.867] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:53.867+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:53.867] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:54.551+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:30:54.551] clusterConfig unchanged [2021-10-22T02:30:54.868+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:54.868] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:55.868+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:55.868] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:56.869+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:56.869] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:57.869+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:57.869] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:58.870+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:58.870] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:30:59.871+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:30:59.871] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:00.873+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:00.873] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:01.874+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:01.874] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:02.874+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:02.874] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:03.212+0000] [.info] [src/director/director.go:computePlan:280] [02:31:03.212] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:31:03.212+0000] [.info] [src/director/director.go:tracef:794] [02:31:03.212] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:31:03.212+0000] [.info] [src/director/director.go:tracef:794] [02:31:03.212] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:31:03.212+0000] [.info] [src/director/director.go:tracef:794] [02:31:03.212] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:31:03.212+0000] [.info] [src/director/director.go:tracef:794] [02:31:03.212] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:31:03.212+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:31:03.212] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:31:03.364+0000] [.info] [src/director/director.go:tracef:794] [02:31:03.364] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:31:03.364+0000] [.info] [src/director/director.go:tracef:794] [02:31:03.364] because [2021-10-22T02:31:03.364+0000] [.info] [src/director/director.go:tracef:794] [02:31:03.364] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:31:03.364+0000] [.info] [src/director/director.go:tracef:794] [02:31:03.364] because ['currentState.CanBounceStop' = true] [2021-10-22T02:31:03.526+0000] [.info] [src/director/director.go:tracef:794] [02:31:03.526] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:31:03.526+0000] [.info] [src/director/director.go:tracef:794] [02:31:03.526] because [2021-10-22T02:31:03.526+0000] [.info] [src/director/director.go:tracef:794] [02:31:03.526] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:31:03.526+0000] [.info] [src/director/director.go:tracef:794] [02:31:03.526] because ['currentState.Up' = false] [2021-10-22T02:31:03.527+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:31:03.527] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:31:03.527+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:31:03.527] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:31:03.610+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:31:03.610] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:31:03.618+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:31:03.618] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:31:03.619+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:31:03.618] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:31:03.619+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:31:03.619] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:31:03.620+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:31:03.620] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:31:03.620+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:31:03.620] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:31:03.620+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:31:03.620] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:31:03.622+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:31:03.622] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:31:03.862+0000] [.info] [src/director/director.go:tracef:794] [02:31:03.862] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:31:03.862+0000] [.info] [src/director/director.go:tracef:794] [02:31:03.862] because ['currentState.Up' = false] [2021-10-22T02:31:03.889+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:03.889] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:04.088+0000] [.info] [src/director/director.go:tracef:794] [02:31:04.088] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:31:04.088+0000] [.info] [src/director/director.go:tracef:794] [02:31:04.088] because ['currentState.Up' = false] [2021-10-22T02:31:04.088+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:31:04.088] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:31:04.327+0000] [.info] [src/director/director.go:tracef:794] [02:31:04.327] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:31:04.327+0000] [.info] [src/director/director.go:tracef:794] [02:31:04.327] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:31:04.535+0000] [.info] [src/director/director.go:tracef:794] [02:31:04.535] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:31:04.535+0000] [.info] [src/director/director.go:tracef:794] [02:31:04.535] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:31:04.535+0000] [.info] [src/director/director.go:tracef:794] [02:31:04.535] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:31:04.535+0000] [.info] [src/director/director.go:tracef:794] [02:31:04.535] because ['currentState.Up' = false] [2021-10-22T02:31:04.535+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:31:04.535] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:31:04.535+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:31:04.535] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:31:04.568+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:31:04.568] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:31:04.569+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:31:04.569] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 43891 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:31:04.743+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:31:04.743] clusterConfig unchanged [2021-10-22T02:31:04.912+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:04.912] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:05.327+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:31:05.327] ...Finished start [pid=26986] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":43891,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:31:05.487+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:31:05.487] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 159.59371ms. [2021-10-22T02:31:05.487+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:31:05.487] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:43891 (local=false) [2021-10-22T02:31:05.487+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:31:05.487] Dropping existing documents from system.replset [2021-10-22T02:31:05.616+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:31:05.616] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:31:05.621+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:31:05.621] Inserting oplogTruncateAfterPoint document [2021-10-22T02:31:05.621+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:31:05.621] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:31:05.622+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:31:05.622] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:31:05.622+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:31:05.622] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:31:05.635+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:31:05.635] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:31:05.635+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:31:05.635] Setting the minvalid document to [{_id ObjectID("61722269523370002becc8b4")} {t -1} {ts {0 1}}] [2021-10-22T02:31:05.647+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:31:05.647] Inserted [{_id ObjectID("61722269523370002becc8b4")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:31:05.647+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:31:05.647] Inserting new replica set configuration document. [2021-10-22T02:31:05.648+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:31:05.648] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:31:05.659+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:31:05.659] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:31:05.673+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:31:05.673] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:31:05.673+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:31:05.673] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:31:05.673+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:31:05.673] Initiating stopEphemeralMongo [2021-10-22T02:31:05.673+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:31:05.673] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:43891 (local=false) [2021-10-22T02:31:05.673+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:31:05.673] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:43891 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:31:05.888+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:05.888] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:05.944+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:31:05.944] clusterConfig unchanged [2021-10-22T02:31:06.007+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:31:06.007] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.860261ms. [2021-10-22T02:31:06.007+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:31:06.007] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:43891 (local=false) [2021-10-22T02:31:06.008+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:31:06.008] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 33.613µs. [2021-10-22T02:31:06.008+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:31:06.008] Successfully finished running stopEphemeralMongo [2021-10-22T02:31:06.008+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:31:06.008] Successfully stopped existing ephemeral mongod. [2021-10-22T02:31:06.008+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:31:06.008] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:31:06.008+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:31:06.008] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:31:06.043+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:31:06.043] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:31:06.044+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:31:06.044] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60220 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:31:06.891+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:06.891] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:07.066+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:31:07.066] clusterConfig unchanged [2021-10-22T02:31:07.245+0000] [.error] [src/action/start.go:func1:93] [103] [02:31:07.245] Error starting mongod : [02:31:07.245] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60220 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 28368 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:31:06.092+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:31:06.092+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:31:06.092+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:31:06.094+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:31:06.096+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:31:06.128+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:31:06.144+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:31:06.144+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:31:06.145+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:31:06.197+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:31:06.197+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:31:06.718+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869866:718322][28368:0x7f6392d3bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:31:06.822+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869866:822424][28368:0x7f6392d3bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:31:06.935+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869866:935479][28368:0x7f6392d3bbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:31:07.053+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869867:53947][28368:0x7f6392d3bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:31:07.128+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869867:128674][28368:0x7f6392d3bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:31:07.189+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869867:189235][28368:0x7f6392d3bbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:31:07.189+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869867:189324][28368:0x7f6392d3bbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:31:07.190+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:31:07.245+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:31:07.245] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60220 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:31:07.245] Error starting mongod : [02:31:07.245] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60220 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 28368 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:31:06.092+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:31:06.092+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:31:06.092+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:31:06.094+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:31:06.096+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:31:06.128+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:31:06.144+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:31:06.144+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:31:06.145+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:31:06.197+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:31:06.197+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:31:06.718+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869866:718322][28368:0x7f6392d3bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:31:06.822+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869866:822424][28368:0x7f6392d3bbc0], txn-recover: [W...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:31:07.245+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:31:07.245] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:31:07.245] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60220 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:31:07.245] Error starting mongod : [02:31:07.245] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60220 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 28368 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:31:06.092+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:31:06.092+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:31:06.092+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:31:06.094+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:31:06.096+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:31:06.128+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:31:06.144+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:31:06.144+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:31:06.145+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:31:06.197+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:31:06.197+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:31:06.718+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869866:718322][28368:0x7f6392d3bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:31:06.822+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ct... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:31:07.246+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:31:07.246] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:31:07.246+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:31:07.246] Initiating stopEphemeralMongo [2021-10-22T02:31:07.246+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:31:07.246] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:31:07.246+0000] [.error] [src/director/director.go:executePlan:962] [02:31:07.246] Failed to apply action. Result = : [02:31:07.245] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:31:07.245] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60220 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:31:07.245] Error starting mongod : [02:31:07.245] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60220 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 28368 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:31:06.092+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:31:06.092+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:31:06.092+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:31:06.094+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:31:06.096+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:31:06.128+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:31:06.144+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:31:06.144+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:31:06.145+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:31:06.197+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:31:06.197+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:31:06.718+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869866:718322][28368:0x7f6392d3bbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:31:07.246+0000] [.error] [src/director/director.go:planAndExecute:567] [02:31:07.246] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:31:07.246] Failed to apply action. Result = : [02:31:07.245] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:31:07.245] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60220 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:31:07.245] Error starting mongod : [02:31:07.245] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60220 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 28368 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:31:06.092+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:31:06.092+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:31:06.092+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:31:06.094+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:31:06.096+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:31:06.128+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:31:06.144+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:31:06.144+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:31:06.145+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:31:06.197+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:31:06.197+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:31:06.718+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:31:07.246+0000] [.error] [src/director/director.go:mainLoop:399] [02:31:07.246] Failed to planAndExecute : [02:31:07.246] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:31:07.246] Failed to apply action. Result = : [02:31:07.245] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:31:07.245] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:60220 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:31:07.245] Error starting mongod : [02:31:07.245] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 60220 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 28368 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:31:06.092+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:31:06.092+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:31:06.092+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":60220,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:31:06.094+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:31:06.096+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:31:06.128+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:31:06.144+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:31:06.144+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:31:06.145+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:31:06.197+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:31:06.197+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:31:06.718+00:00"},"s":"I", "c":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:31:07.891+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:07.891] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:08.327+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:31:08.327] clusterConfig unchanged [2021-10-22T02:31:08.891+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:08.891] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:09.892+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:09.892] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:10.892+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:10.892] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:11.892+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:11.892] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:12.892+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:12.892] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:13.893+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:13.893] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:14.893+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:14.893] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:15.893+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:15.893] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:16.637+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:31:16.637] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 13.015470126s [2021-10-22T02:31:16.891+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:16.891] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:17.891+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:17.891] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:18.537+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:31:18.537] clusterConfig unchanged [2021-10-22T02:31:18.566+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:31:18.566] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:31:18.568+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:18.568] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:31:18.568+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [02:31:18.568] Could not get version : [02:31:18.568] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:31:18.568+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [02:31:18.568] Error getting version for log lines. Skipping : [02:31:18.568] Could not get version : [02:31:18.568] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:31:18.899+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:18.899] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:19.891+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:19.891] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:20.892+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:20.892] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:21.893+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:21.893] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:22.893+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:22.893] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:23.894+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:23.894] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:24.896+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:24.896] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:25.896+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:25.896] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:26.896+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:26.896] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:27.897+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:27.897] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:28.639+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:31:28.639] clusterConfig unchanged [2021-10-22T02:31:28.897+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:28.897] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:29.898+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:29.898] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:30.899+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:30.899] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:31.899+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:31.899] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:32.899+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:32.899] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:33.900+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:33.900] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:34.899+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:34.899] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:35.900+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:35.900] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:36.900+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:36.900] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:37.900+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:37.900] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:38.761+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:31:38.761] clusterConfig unchanged [2021-10-22T02:31:38.901+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:38.901] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:39.902+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:39.902] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:40.904+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:40.904] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:41.408+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:31:41.408] memory:rss=57769984 alloc=8418112 totalalloc=11306366824 mallocs=144353674 frees=144295753 heapinuse=16932864 heapobjects=57921 gcpausetotalns=1648405791 gcpauselastns=59309 gcnum=1648 utimens=248036703000 stimens=99179829000 minflt=610124 majflt=405 [2021-10-22T02:31:41.904+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:41.904] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:42.905+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:42.905] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:43.905+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:43.905] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:44.905+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:44.905] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:45.906+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:45.906] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:46.907+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:46.907] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:47.907+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:47.907] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:48.892+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:31:48.892] clusterConfig unchanged [2021-10-22T02:31:48.907+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:48.907] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:49.910+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:49.910] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:50.909+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:50.909] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:51.909+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:51.909] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:52.912+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:52.912] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:53.911+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:53.911] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:54.912+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:54.912] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:55.912+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:55.912] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:56.913+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:56.913] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:57.912+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:57.912] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:58.913+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:58.913] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:31:59.014+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:31:59.014] clusterConfig unchanged [2021-10-22T02:31:59.915+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:31:59.915] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:00.915+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:00.915] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:01.918+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:01.918] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:02.916+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:02.916] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:03.916+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:03.916] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:04.917+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:04.917] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:05.917+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:05.917] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:06.918+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:06.918] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:07.424+0000] [.info] [src/director/director.go:computePlan:280] [02:32:07.424] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:32:07.424+0000] [.info] [src/director/director.go:tracef:794] [02:32:07.424] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:32:07.424+0000] [.info] [src/director/director.go:tracef:794] [02:32:07.424] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:32:07.424+0000] [.info] [src/director/director.go:tracef:794] [02:32:07.424] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:32:07.424+0000] [.info] [src/director/director.go:tracef:794] [02:32:07.424] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:32:07.425+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:32:07.425] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:32:07.588+0000] [.info] [src/director/director.go:tracef:794] [02:32:07.588] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:32:07.588+0000] [.info] [src/director/director.go:tracef:794] [02:32:07.588] because [2021-10-22T02:32:07.588+0000] [.info] [src/director/director.go:tracef:794] [02:32:07.588] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:32:07.588+0000] [.info] [src/director/director.go:tracef:794] [02:32:07.588] because ['currentState.CanBounceStop' = true] [2021-10-22T02:32:07.738+0000] [.info] [src/director/director.go:tracef:794] [02:32:07.738] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:32:07.738+0000] [.info] [src/director/director.go:tracef:794] [02:32:07.738] because [2021-10-22T02:32:07.739+0000] [.info] [src/director/director.go:tracef:794] [02:32:07.739] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:32:07.739+0000] [.info] [src/director/director.go:tracef:794] [02:32:07.739] because ['currentState.Up' = false] [2021-10-22T02:32:07.739+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:32:07.739] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:32:07.739+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:32:07.739] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:32:07.817+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:32:07.817] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:32:07.825+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:32:07.825] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:32:07.826+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:32:07.826] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:32:07.827+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:32:07.827] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:32:07.827+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:32:07.827] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:32:07.827+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:32:07.827] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:32:07.828+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:32:07.828] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:32:07.924+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:07.924] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:08.066+0000] [.info] [src/director/director.go:tracef:794] [02:32:08.066] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:32:08.066+0000] [.info] [src/director/director.go:tracef:794] [02:32:08.066] because ['currentState.Up' = false] [2021-10-22T02:32:08.305+0000] [.info] [src/director/director.go:tracef:794] [02:32:08.305] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:32:08.305+0000] [.info] [src/director/director.go:tracef:794] [02:32:08.305] because ['currentState.Up' = false] [2021-10-22T02:32:08.305+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:32:08.305] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:32:08.545+0000] [.info] [src/director/director.go:tracef:794] [02:32:08.545] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:32:08.545+0000] [.info] [src/director/director.go:tracef:794] [02:32:08.545] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:32:08.762+0000] [.info] [src/director/director.go:tracef:794] [02:32:08.762] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:32:08.762+0000] [.info] [src/director/director.go:tracef:794] [02:32:08.762] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:32:08.762+0000] [.info] [src/director/director.go:tracef:794] [02:32:08.762] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:32:08.762+0000] [.info] [src/director/director.go:tracef:794] [02:32:08.762] because ['currentState.Up' = false] [2021-10-22T02:32:08.762+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:32:08.762] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:32:08.762+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:32:08.762] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:32:08.795+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:32:08.794] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:32:08.795+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:32:08.795] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 46847 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:32:08.923+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:08.923] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:09.230+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:32:09.230] clusterConfig unchanged [2021-10-22T02:32:09.521+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:32:09.521] ...Finished start [pid=47857] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":46847,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:32:09.650+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:32:09.650] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 128.618104ms. [2021-10-22T02:32:09.650+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:32:09.650] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46847 (local=false) [2021-10-22T02:32:09.650+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:32:09.650] Dropping existing documents from system.replset [2021-10-22T02:32:09.775+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:32:09.775] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:32:09.796+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:32:09.796] Inserting oplogTruncateAfterPoint document [2021-10-22T02:32:09.796+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:32:09.796] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:32:09.797+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:32:09.797] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:32:09.797+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:32:09.797] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:32:09.812+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:32:09.812] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:32:09.812+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:32:09.812] Setting the minvalid document to [{_id ObjectID("617222a9523370002becc8c2")} {t -1} {ts {0 1}}] [2021-10-22T02:32:09.835+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:32:09.835] Inserted [{_id ObjectID("617222a9523370002becc8c2")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:32:09.835+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:32:09.835] Inserting new replica set configuration document. [2021-10-22T02:32:09.835+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:32:09.835] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:32:09.858+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:32:09.858] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:32:09.861+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:32:09.861] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:32:09.861+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:32:09.861] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:32:09.861+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:32:09.861] Initiating stopEphemeralMongo [2021-10-22T02:32:09.861+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:32:09.861] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46847 (local=false) [2021-10-22T02:32:09.861+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:32:09.861] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46847 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:32:09.923+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:09.923] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:10.241+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:32:10.241] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.129507ms. [2021-10-22T02:32:10.241+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:32:10.241] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:46847 (local=false) [2021-10-22T02:32:10.241+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:32:10.241] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 39.25µs. [2021-10-22T02:32:10.241+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:32:10.241] Successfully finished running stopEphemeralMongo [2021-10-22T02:32:10.241+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:32:10.241] Successfully stopped existing ephemeral mongod. [2021-10-22T02:32:10.242+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:32:10.241] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:32:10.242+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:32:10.242] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:32:10.275+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:32:10.275] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:32:10.276+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:32:10.276] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41703 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:32:10.348+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:32:10.348] clusterConfig unchanged [2021-10-22T02:32:10.932+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:10.932] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:11.491+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:32:11.491] clusterConfig unchanged [2021-10-22T02:32:11.527+0000] [.error] [src/action/start.go:func1:93] [103] [02:32:11.527] Error starting mongod : [02:32:11.527] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41703,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41703 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 49246 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:32:10.327+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:32:10.328+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:32:10.328+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41703,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:32:10.329+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:32:10.331+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:32:10.332+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:32:10.389+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:32:10.389+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:32:10.389+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:32:10.437+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:32:10.437+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:32:10.995+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869930:995790][49246:0x7f6f91d7ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:32:11.102+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869931:102346][49246:0x7f6f91d7ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:32:11.209+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869931:209924][49246:0x7f6f91d7ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:32:11.328+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869931:328754][49246:0x7f6f91d7ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:32:11.408+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869931:408169][49246:0x7f6f91d7ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:32:11.475+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869931:475485][49246:0x7f6f91d7ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:32:11.475+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869931:475557][49246:0x7f6f91d7ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:32:11.477+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:32:11.527+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:32:11.527] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:41703 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:32:11.527] Error starting mongod : [02:32:11.527] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41703,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41703 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 49246 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:32:10.327+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:32:10.328+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:32:10.328+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41703,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:32:10.329+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:32:10.331+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:32:10.332+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:32:10.389+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:32:10.389+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:32:10.389+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:32:10.437+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:32:10.437+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:32:10.995+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869930:995790][49246:0x7f6f91d7ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:32:11.102+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869931:102346][49246:0x7f6f91d7ebc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:32:11.528+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:32:11.527] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:32:11.527] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:41703 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:32:11.527] Error starting mongod : [02:32:11.527] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41703,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41703 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 49246 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:32:10.327+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:32:10.328+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:32:10.328+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41703,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:32:10.329+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:32:10.331+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:32:10.332+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:32:10.389+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:32:10.389+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:32:10.389+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:32:10.437+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:32:10.437+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:32:10.995+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869930:995790][49246:0x7f6f91d7ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:32:11.102+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:32:11.528+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:32:11.528] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:32:11.528+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:32:11.528] Initiating stopEphemeralMongo [2021-10-22T02:32:11.528+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:32:11.528] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:32:11.529+0000] [.error] [src/director/director.go:executePlan:962] [02:32:11.529] Failed to apply action. Result = : [02:32:11.527] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:32:11.527] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:41703 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:32:11.527] Error starting mongod : [02:32:11.527] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41703,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41703 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 49246 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:32:10.327+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:32:10.328+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:32:10.328+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41703,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:32:10.329+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:32:10.331+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:32:10.332+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:32:10.389+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:32:10.389+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:32:10.389+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:32:10.437+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:32:10.437+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:32:10.995+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869930:995790][49246:0x7f6f91d7ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:32:11.529+0000] [.error] [src/director/director.go:planAndExecute:567] [02:32:11.529] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:32:11.529] Failed to apply action. Result = : [02:32:11.527] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:32:11.527] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:41703 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:32:11.527] Error starting mongod : [02:32:11.527] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41703,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41703 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 49246 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:32:10.327+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:32:10.328+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:32:10.328+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41703,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:32:10.329+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:32:10.331+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:32:10.332+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:32:10.389+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:32:10.389+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:32:10.389+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:32:10.437+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:32:10.437+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:32:10.995+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:32:11.529+0000] [.error] [src/director/director.go:mainLoop:399] [02:32:11.529] Failed to planAndExecute : [02:32:11.529] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:32:11.529] Failed to apply action. Result = : [02:32:11.527] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:32:11.527] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:41703 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:32:11.527] Error starting mongod : [02:32:11.527] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41703,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 41703 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 49246 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:32:10.327+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:32:10.328+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:32:10.328+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":41703,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:32:10.329+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:32:10.331+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:32:10.332+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:32:10.389+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:32:10.389+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:32:10.389+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:32:10.437+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:32:10.437+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:32:10.995+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:32:11.933+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:11.933] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:12.730+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:32:12.730] clusterConfig unchanged [2021-10-22T02:32:12.931+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:12.931] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:13.933+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:13.933] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:14.932+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:14.932] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:15.934+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:15.934] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:16.934+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:16.934] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:17.932+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:17.932] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:18.934+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:18.934] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:19.934+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:19.934] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:20.724+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:32:20.724] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.895671346s [2021-10-22T02:32:20.932+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:20.932] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:21.933+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:21.933] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:22.832+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:32:22.832] clusterConfig unchanged [2021-10-22T02:32:22.934+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:22.934] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:23.935+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:23.935] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:24.942+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:24.942] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:25.935+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:25.935] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:26.936+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:26.936] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:27.556+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:32:27.556] ClientMetadata Mspooler loop timeout after: 1h0m0s. Flushing to mms [2021-10-22T02:32:27.558+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:27.557] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:32:27.558+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [02:32:27.558] Could not get version : [02:32:27.557] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:32:27.558+0000] [.warn] [filebeat/mspooler/clientmetadata.go:PreFlushProcessing:192] [02:32:27.558] Error getting version for metadata entry. Skipping : [02:32:27.558] Could not get version : [02:32:27.557] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:32:27.572+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:32:27.572] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:32:27.572+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:32:27.572] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:32:27.941+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:27.941] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:28.939+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:28.939] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:29.939+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:29.939] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:30.940+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:30.940] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:31.941+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:31.941] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:32.941+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:32.941] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:32.944+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:32:32.944] clusterConfig unchanged [2021-10-22T02:32:33.942+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:33.942] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:34.943+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:34.943] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:35.944+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:35.944] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:36.943+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:36.943] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:37.944+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:37.944] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:38.946+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [02:32:38.946] lsblk cache invalidated, running lsblk command [2021-10-22T02:32:38.947+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:38.947] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:39.945+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:39.945] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:40.945+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:40.945] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:41.409+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:32:41.409] memory:rss=56344576 alloc=9658856 totalalloc=11400554088 mallocs=145557508 frees=145485699 heapinuse=17113088 heapobjects=71809 gcpausetotalns=1649589630 gcpauselastns=56415 gcnum=1660 utimens=250604128000 stimens=100371517000 minflt=615529 majflt=405 [2021-10-22T02:32:41.945+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:41.945] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:42.946+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:42.946] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:43.096+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:32:43.096] clusterConfig unchanged [2021-10-22T02:32:43.949+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:43.949] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:44.450+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [02:32:44.450] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:41703 (local=false) because it is not up [2021-10-22T02:32:44.946+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:44.946] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:45.949+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:45.949] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:46.953+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:46.953] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:47.964+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:47.964] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:48.949+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:48.948] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:49.949+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:49.949] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:50.950+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:50.950] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:51.953+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:51.953] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:52.951+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:52.951] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:53.217+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:32:53.217] clusterConfig unchanged [2021-10-22T02:32:53.952+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:53.952] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:54.953+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:54.953] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:55.955+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:55.955] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:56.953+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:56.953] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:57.954+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:57.954] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:58.958+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:58.958] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:32:59.956+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:32:59.956] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:00.955+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:00.955] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:01.961+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:01.961] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:02.957+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:02.957] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:03.329+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:33:03.329] clusterConfig unchanged [2021-10-22T02:33:03.957+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:03.957] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:04.958+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:04.958] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:05.958+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:05.958] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:06.959+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:06.959] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:07.964+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:07.964] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:08.962+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:08.962] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:09.967+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:09.967] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:10.966+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:10.966] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:11.702+0000] [.info] [src/director/director.go:computePlan:280] [02:33:11.702] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:33:11.703+0000] [.info] [src/director/director.go:tracef:794] [02:33:11.703] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:33:11.703+0000] [.info] [src/director/director.go:tracef:794] [02:33:11.703] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:33:11.703+0000] [.info] [src/director/director.go:tracef:794] [02:33:11.703] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:33:11.703+0000] [.info] [src/director/director.go:tracef:794] [02:33:11.703] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:33:11.703+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:33:11.703] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:33:11.856+0000] [.info] [src/director/director.go:tracef:794] [02:33:11.856] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:33:11.856+0000] [.info] [src/director/director.go:tracef:794] [02:33:11.856] because [2021-10-22T02:33:11.856+0000] [.info] [src/director/director.go:tracef:794] [02:33:11.856] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:33:11.856+0000] [.info] [src/director/director.go:tracef:794] [02:33:11.856] because ['currentState.CanBounceStop' = true] [2021-10-22T02:33:11.965+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:11.965] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:12.014+0000] [.info] [src/director/director.go:tracef:794] [02:33:12.014] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:33:12.014+0000] [.info] [src/director/director.go:tracef:794] [02:33:12.014] because [2021-10-22T02:33:12.015+0000] [.info] [src/director/director.go:tracef:794] [02:33:12.015] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:33:12.015+0000] [.info] [src/director/director.go:tracef:794] [02:33:12.015] because ['currentState.Up' = false] [2021-10-22T02:33:12.015+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:33:12.015] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:33:12.015+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:33:12.015] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:33:12.092+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:33:12.092] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:33:12.100+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:33:12.100] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:33:12.101+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:33:12.101] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:33:12.102+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:33:12.102] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:33:12.103+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:33:12.103] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:33:12.103+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:33:12.103] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:33:12.103+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:33:12.103] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:33:12.104+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:33:12.104] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:33:12.355+0000] [.info] [src/director/director.go:tracef:794] [02:33:12.355] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:33:12.355+0000] [.info] [src/director/director.go:tracef:794] [02:33:12.355] because ['currentState.Up' = false] [2021-10-22T02:33:12.559+0000] [.info] [src/director/director.go:tracef:794] [02:33:12.559] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:33:12.559+0000] [.info] [src/director/director.go:tracef:794] [02:33:12.559] because ['currentState.Up' = false] [2021-10-22T02:33:12.559+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:33:12.559] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:33:12.783+0000] [.info] [src/director/director.go:tracef:794] [02:33:12.783] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:33:12.783+0000] [.info] [src/director/director.go:tracef:794] [02:33:12.783] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:33:12.975+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:12.975] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:13.003+0000] [.info] [src/director/director.go:tracef:794] [02:33:13.003] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:33:13.003+0000] [.info] [src/director/director.go:tracef:794] [02:33:13.003] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:33:13.003+0000] [.info] [src/director/director.go:tracef:794] [02:33:13.003] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:33:13.003+0000] [.info] [src/director/director.go:tracef:794] [02:33:13.003] because ['currentState.Up' = false] [2021-10-22T02:33:13.003+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:33:13.003] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:33:13.003+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:33:13.003] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:33:13.039+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:33:13.039] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:33:13.040+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:33:13.040] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 45924 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:33:13.510+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:33:13.510] clusterConfig unchanged [2021-10-22T02:33:13.793+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:33:13.793] ...Finished start [pid=68489] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":45924,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:33:13.920+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:33:13.919] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 126.603474ms. [2021-10-22T02:33:13.920+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:33:13.920] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45924 (local=false) [2021-10-22T02:33:13.920+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:33:13.920] Dropping existing documents from system.replset [2021-10-22T02:33:13.976+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:13.976] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:14.070+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:33:14.070] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:33:14.075+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:33:14.075] Inserting oplogTruncateAfterPoint document [2021-10-22T02:33:14.075+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:33:14.075] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:33:14.076+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:33:14.076] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:33:14.076+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:33:14.076] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:33:14.089+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:33:14.089] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:33:14.089+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:33:14.089] Setting the minvalid document to [{_id ObjectID("617222ea523370002becc8d0")} {t -1} {ts {0 1}}] [2021-10-22T02:33:14.112+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:33:14.112] Inserted [{_id ObjectID("617222ea523370002becc8d0")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:33:14.113+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:33:14.112] Inserting new replica set configuration document. [2021-10-22T02:33:14.113+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:33:14.113] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:33:14.137+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:33:14.137] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:33:14.141+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:33:14.141] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:33:14.141+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:33:14.141] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:33:14.141+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:33:14.141] Initiating stopEphemeralMongo [2021-10-22T02:33:14.141+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:33:14.141] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45924 (local=false) [2021-10-22T02:33:14.141+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:33:14.141] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45924 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:33:14.541+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:33:14.541] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.294057ms. [2021-10-22T02:33:14.541+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:33:14.541] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:45924 (local=false) [2021-10-22T02:33:14.542+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:33:14.542] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.399µs. [2021-10-22T02:33:14.542+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:33:14.542] Successfully finished running stopEphemeralMongo [2021-10-22T02:33:14.542+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:33:14.542] Successfully stopped existing ephemeral mongod. [2021-10-22T02:33:14.542+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:33:14.542] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:33:14.542+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:33:14.542] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:33:14.575+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:33:14.575] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:33:14.576+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:33:14.576] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56054 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:33:14.639+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:33:14.638] clusterConfig unchanged [2021-10-22T02:33:14.971+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:14.971] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:15.756+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:33:15.756] clusterConfig unchanged [2021-10-22T02:33:15.794+0000] [.error] [src/action/start.go:func1:93] [103] [02:33:15.794] Error starting mongod : [02:33:15.794] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56054,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56054 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:33:14.607Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 69978 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:33:14.639+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:33:14.639+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:33:14.639+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56054,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:33:14.641+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:33:14.641+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:33:14.643+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:33:14.697+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:33:14.697+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:33:14.698+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:33:14.750+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:33:14.750+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:33:15.262+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869995:262473][69978:0x7f3d1b8d0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:33:15.374+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869995:374899][69978:0x7f3d1b8d0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:33:15.491+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869995:491037][69978:0x7f3d1b8d0bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T02:33:15.609+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869995:609497][69978:0x7f3d1b8d0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:33:15.682+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869995:682156][69978:0x7f3d1b8d0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:33:15.741+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869995:741792][69978:0x7f3d1b8d0bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:33:15.741+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869995:741871][69978:0x7f3d1b8d0bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:33:15.794+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:33:15.794] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:56054 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:33:15.794] Error starting mongod : [02:33:15.794] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56054,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56054 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:33:14.607Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 69978 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:33:14.639+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:33:14.639+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:33:14.639+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56054,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:33:14.641+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:33:14.641+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:33:14.643+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:33:14.697+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:33:14.697+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:33:14.698+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:33:14.750+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:33:14.750+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:33:15.262+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869995:262473][69978:0x7f3d1b8d0bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:33:15.374+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:33:15.795+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:33:15.795] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:33:15.794] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:56054 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:33:15.794] Error starting mongod : [02:33:15.794] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56054,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56054 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:33:14.607Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 69978 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:33:14.639+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:33:14.639+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:33:14.639+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56054,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:33:14.641+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:33:14.641+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:33:14.643+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:33:14.697+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:33:14.697+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:33:14.698+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:33:14.750+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:33:14.750+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:33:15.262+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634869995:262473][69978:0x7f3d1b8d0bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:33:15.795+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:33:15.795] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:33:15.795+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:33:15.795] Initiating stopEphemeralMongo [2021-10-22T02:33:15.795+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:33:15.795] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:33:15.796+0000] [.error] [src/director/director.go:executePlan:962] [02:33:15.796] Failed to apply action. Result = : [02:33:15.795] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:33:15.794] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:56054 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:33:15.794] Error starting mongod : [02:33:15.794] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56054,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56054 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:33:14.607Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 69978 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:33:14.639+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:33:14.639+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:33:14.639+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56054,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:33:14.641+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:33:14.641+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:33:14.643+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:33:14.697+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:33:14.697+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:33:14.698+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:33:14.750+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:33:14.750+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:33:15.262+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:33:15.796+0000] [.error] [src/director/director.go:planAndExecute:567] [02:33:15.796] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:33:15.796] Failed to apply action. Result = : [02:33:15.795] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:33:15.794] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:56054 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:33:15.794] Error starting mongod : [02:33:15.794] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56054,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56054 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:33:14.607Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 69978 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:33:14.639+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:33:14.639+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:33:14.639+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56054,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:33:14.641+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:33:14.641+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:33:14.643+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:33:14.697+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:33:14.697+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:33:14.698+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:33:14.750+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:33:14.750+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:33:15.796+0000] [.error] [src/director/director.go:mainLoop:399] [02:33:15.796] Failed to planAndExecute : [02:33:15.796] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:33:15.796] Failed to apply action. Result = : [02:33:15.795] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:33:15.794] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:56054 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:33:15.794] Error starting mongod : [02:33:15.794] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56054,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 56054 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:33:14.607Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 69978 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:33:14.639+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:33:14.639+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:33:14.639+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":56054,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:33:14.641+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:33:14.641+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:33:14.643+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:33:14.697+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:33:14.697+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:33:14.698+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:33:14.750+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:33:14.750+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:33:15.972+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:15.972] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:16.973+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:16.973] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:17.006+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:33:17.006] clusterConfig unchanged [2021-10-22T02:33:17.972+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:17.972] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:18.991+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:18.991] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:19.974+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:19.974] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:20.974+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:20.974] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:21.974+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:21.974] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:22.975+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:22.975] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:23.976+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:23.976] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:24.976+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:24.976] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:25.092+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:33:25.092] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.98766753s [2021-10-22T02:33:25.978+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:25.978] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:26.978+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:26.978] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:27.117+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:33:27.117] clusterConfig unchanged [2021-10-22T02:33:27.976+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:27.976] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:28.979+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:28.979] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:29.983+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:29.983] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:30.980+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:30.980] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:31.981+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:31.981] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:32.981+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:32.981] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:33.983+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:33.983] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:34.983+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:34.983] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:35.984+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:35.984] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:36.983+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:36.983] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:37.239+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:33:37.239] clusterConfig unchanged [2021-10-22T02:33:37.983+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:37.983] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:38.982+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:38.982] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:39.985+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:39.985] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:40.984+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:40.984] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:41.410+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:33:41.410] memory:rss=58437632 alloc=8393624 totalalloc=11493680304 mallocs=146764488 frees=146707901 heapinuse=17129472 heapobjects=56587 gcpausetotalns=1650696779 gcpauselastns=56255 gcnum=1672 utimens=253813533000 stimens=101951339000 minflt=621560 majflt=405 [2021-10-22T02:33:41.987+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:41.987] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:42.985+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:42.985] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:43.989+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:43.989] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:44.990+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:44.990] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:45.988+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:45.988] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:46.988+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:46.988] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:47.351+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:33:47.351] clusterConfig unchanged [2021-10-22T02:33:47.988+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:47.988] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:48.989+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:48.989] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:49.992+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:49.992] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:50.992+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:50.992] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:51.991+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:51.991] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:52.992+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:52.992] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:53.993+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:53.993] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:54.993+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:54.993] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:55.996+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:55.996] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:56.994+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:56.994] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:57.443+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:33:57.443] clusterConfig unchanged [2021-10-22T02:33:57.996+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:57.996] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:58.996+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:58.996] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:33:59.997+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:33:59.997] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:00.998+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:00.997] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:02.006+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:02.006] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:03.002+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:03.002] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:03.999+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:03.999] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:05.000+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:05.000] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:06.000+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:06.000] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:07.000+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:07.000] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:07.536+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:34:07.536] clusterConfig unchanged [2021-10-22T02:34:08.001+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:08.001] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:09.018+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:09.018] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:10.002+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:10.002] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:11.002+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:11.002] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:12.003+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:12.003] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:13.010+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:13.010] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:14.004+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:14.004] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:15.012+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:15.012] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:16.024+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:16.024] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:16.084+0000] [.info] [src/director/director.go:computePlan:280] [02:34:16.084] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:34:16.085+0000] [.info] [src/director/director.go:tracef:794] [02:34:16.084] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:34:16.085+0000] [.info] [src/director/director.go:tracef:794] [02:34:16.085] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:34:16.085+0000] [.info] [src/director/director.go:tracef:794] [02:34:16.085] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:34:16.085+0000] [.info] [src/director/director.go:tracef:794] [02:34:16.085] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:34:16.085+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:34:16.085] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:34:16.236+0000] [.info] [src/director/director.go:tracef:794] [02:34:16.236] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:34:16.236+0000] [.info] [src/director/director.go:tracef:794] [02:34:16.236] because [2021-10-22T02:34:16.236+0000] [.info] [src/director/director.go:tracef:794] [02:34:16.236] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:34:16.236+0000] [.info] [src/director/director.go:tracef:794] [02:34:16.236] because ['currentState.CanBounceStop' = true] [2021-10-22T02:34:16.402+0000] [.info] [src/director/director.go:tracef:794] [02:34:16.402] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:34:16.402+0000] [.info] [src/director/director.go:tracef:794] [02:34:16.402] because [2021-10-22T02:34:16.402+0000] [.info] [src/director/director.go:tracef:794] [02:34:16.402] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:34:16.402+0000] [.info] [src/director/director.go:tracef:794] [02:34:16.402] because ['currentState.Up' = false] [2021-10-22T02:34:16.402+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:34:16.402] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:34:16.402+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:34:16.402] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:34:16.487+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:34:16.487] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:34:16.495+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:34:16.495] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:34:16.495+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:34:16.495] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:34:16.496+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:34:16.496] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:34:16.497+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:34:16.497] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:34:16.497+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:34:16.497] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:34:16.497+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:34:16.497] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:34:16.498+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:34:16.498] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:34:16.721+0000] [.info] [src/director/director.go:tracef:794] [02:34:16.721] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:34:16.721+0000] [.info] [src/director/director.go:tracef:794] [02:34:16.721] because ['currentState.Up' = false] [2021-10-22T02:34:16.939+0000] [.info] [src/director/director.go:tracef:794] [02:34:16.939] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:34:16.939+0000] [.info] [src/director/director.go:tracef:794] [02:34:16.939] because ['currentState.Up' = false] [2021-10-22T02:34:16.939+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:34:16.939] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:34:17.031+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:17.031] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:17.166+0000] [.info] [src/director/director.go:tracef:794] [02:34:17.166] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:34:17.166+0000] [.info] [src/director/director.go:tracef:794] [02:34:17.166] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:34:17.388+0000] [.info] [src/director/director.go:tracef:794] [02:34:17.388] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:34:17.388+0000] [.info] [src/director/director.go:tracef:794] [02:34:17.388] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:34:17.388+0000] [.info] [src/director/director.go:tracef:794] [02:34:17.388] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:34:17.388+0000] [.info] [src/director/director.go:tracef:794] [02:34:17.388] because ['currentState.Up' = false] [2021-10-22T02:34:17.388+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:34:17.388] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:34:17.388+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:34:17.388] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:34:17.422+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:34:17.422] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:34:17.422+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:34:17.422] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 58746 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:34:17.707+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:34:17.707] clusterConfig unchanged [2021-10-22T02:34:18.020+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:18.020] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:18.140+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:34:18.140] ...Finished start [pid=15770] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":58746,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:34:18.275+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:34:18.275] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 134.236544ms. [2021-10-22T02:34:18.275+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:34:18.275] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:58746 (local=false) [2021-10-22T02:34:18.275+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:34:18.275] Dropping existing documents from system.replset [2021-10-22T02:34:18.404+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:34:18.404] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:34:18.419+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:34:18.419] Inserting oplogTruncateAfterPoint document [2021-10-22T02:34:18.419+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:34:18.419] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:34:18.420+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:34:18.420] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:34:18.420+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:34:18.420] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:34:18.433+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:34:18.433] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:34:18.433+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:34:18.433] Setting the minvalid document to [{_id ObjectID("6172232a523370002becc8de")} {t -1} {ts {0 1}}] [2021-10-22T02:34:18.457+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:34:18.457] Inserted [{_id ObjectID("6172232a523370002becc8de")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:34:18.457+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:34:18.457] Inserting new replica set configuration document. [2021-10-22T02:34:18.457+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:34:18.457] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:34:18.473+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:34:18.473] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:34:18.477+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:34:18.477] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:34:18.477+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:34:18.477] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:34:18.477+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:34:18.477] Initiating stopEphemeralMongo [2021-10-22T02:34:18.477+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:34:18.477] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:58746 (local=false) [2021-10-22T02:34:18.477+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:34:18.477] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:58746 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:34:18.810+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:34:18.810] clusterConfig unchanged [2021-10-22T02:34:18.869+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:34:18.869] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.375437ms. [2021-10-22T02:34:18.869+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:34:18.869] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:58746 (local=false) [2021-10-22T02:34:18.869+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:34:18.869] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 30.037µs. [2021-10-22T02:34:18.869+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:34:18.869] Successfully finished running stopEphemeralMongo [2021-10-22T02:34:18.869+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:34:18.869] Successfully stopped existing ephemeral mongod. [2021-10-22T02:34:18.869+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:34:18.869] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:34:18.869+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:34:18.869] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:34:18.903+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:34:18.903] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:34:18.904+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:34:18.904] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57966 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:34:19.020+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:19.020] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:19.904+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:34:19.904] clusterConfig unchanged [2021-10-22T02:34:20.030+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:20.029] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:20.078+0000] [.error] [src/action/start.go:func1:93] [103] [02:34:20.078] Error starting mongod : [02:34:20.078] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57966,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57966 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 17302 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:34:18.964+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:34:18.964+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:34:18.964+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57966,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:34:18.967+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:34:18.968+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:34:18.971+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:34:19.076+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:34:19.077+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:34:19.094+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:34:19.137+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:34:19.137+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:34:19.569+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870059:569085][17302:0x7efc1b12fbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:34:19.673+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870059:673012][17302:0x7efc1b12fbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:34:19.773+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870059:773772][17302:0x7efc1b12fbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:34:19.890+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870059:890848][17302:0x7efc1b12fbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:34:19.964+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870059:964228][17302:0x7efc1b12fbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:34:20.028+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870060:28472][17302:0x7efc1b12fbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:34:20.028+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870060:28537][17302:0x7efc1b12fbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:34:20.030+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"in...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:34:20.078+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:34:20.078] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:57966 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:34:20.078] Error starting mongod : [02:34:20.078] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57966,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57966 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 17302 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:34:18.964+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:34:18.964+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:34:18.964+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57966,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:34:18.967+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:34:18.968+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:34:18.971+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:34:19.076+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:34:19.077+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:34:19.094+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:34:19.137+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:34:19.137+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:34:19.569+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870059:569085][17302:0x7efc1b12fbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:34:19.673+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870059:673012][17302:0x7efc1b12fbc0], txn-recover: ...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:34:20.078+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:34:20.078] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:34:20.078] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:57966 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:34:20.078] Error starting mongod : [02:34:20.078] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57966,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57966 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 17302 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:34:18.964+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:34:18.964+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:34:18.964+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57966,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:34:18.967+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:34:18.968+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:34:18.971+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:34:19.076+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:34:19.077+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:34:19.094+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:34:19.137+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:34:19.137+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:34:19.569+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870059:569085][17302:0x7efc1b12fbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:34:19.673+00:00"},"s":"I", "c":"STORAGE", "id":22430, "... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:34:20.079+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:34:20.078] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:34:20.079+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:34:20.079] Initiating stopEphemeralMongo [2021-10-22T02:34:20.079+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:34:20.079] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:34:20.079+0000] [.error] [src/director/director.go:executePlan:962] [02:34:20.079] Failed to apply action. Result = : [02:34:20.078] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:34:20.078] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:57966 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:34:20.078] Error starting mongod : [02:34:20.078] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57966,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57966 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 17302 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:34:18.964+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:34:18.964+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:34:18.964+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57966,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:34:18.967+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:34:18.968+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:34:18.971+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:34:19.076+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:34:19.077+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:34:19.094+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:34:19.137+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:34:19.137+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:34:19.569+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870059:569085][17302:0x7efc1b12fbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:34:20.079+0000] [.error] [src/director/director.go:planAndExecute:567] [02:34:20.079] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:34:20.079] Failed to apply action. Result = : [02:34:20.078] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:34:20.078] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:57966 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:34:20.078] Error starting mongod : [02:34:20.078] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57966,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57966 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 17302 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:34:18.964+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:34:18.964+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:34:18.964+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57966,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:34:18.967+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:34:18.968+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:34:18.971+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:34:19.076+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:34:19.077+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:34:19.094+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:34:19.137+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:34:19.137+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:34:19.569+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger messa... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:34:20.079+0000] [.error] [src/director/director.go:mainLoop:399] [02:34:20.079] Failed to planAndExecute : [02:34:20.079] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:34:20.079] Failed to apply action. Result = : [02:34:20.078] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:34:20.078] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:57966 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:34:20.078] Error starting mongod : [02:34:20.078] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57966,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57966 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 17302 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:34:18.964+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:34:18.964+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:34:18.964+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57966,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:34:18.967+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:34:18.968+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:34:18.971+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:34:19.076+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:34:19.077+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:34:19.094+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:34:19.137+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:34:19.137+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:34:19.569+00:00"},"s":"I", "c... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:34:21.029+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:21.029] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:21.173+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:34:21.173] clusterConfig unchanged [2021-10-22T02:34:22.049+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:22.049] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:23.028+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:23.028] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:24.030+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:24.030] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:25.034+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:25.034] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:26.031+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:26.031] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:27.032+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:27.032] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:28.033+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:28.033] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:29.034+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:29.034] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:29.242+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:34:29.242] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.743897846s [2021-10-22T02:34:30.030+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:30.030] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:31.031+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:31.031] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:31.265+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:34:31.265] clusterConfig unchanged [2021-10-22T02:34:32.032+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:32.032] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:33.040+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:33.040] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:34.034+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:34.034] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:35.034+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:35.034] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:36.034+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:36.034] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:37.035+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:37.035] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:38.036+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:38.036] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:39.036+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:39.036] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:40.037+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:40.037] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:41.037+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:41.037] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:41.411+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:34:41.411] memory:rss=60510208 alloc=14027000 totalalloc=11587433312 mallocs=147971401 frees=147850421 heapinuse=19202048 heapobjects=120980 gcpausetotalns=1652038444 gcpauselastns=139931 gcnum=1683 utimens=256393931000 stimens=103145602000 minflt=627024 majflt=405 [2021-10-22T02:34:41.446+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:34:41.446] clusterConfig unchanged [2021-10-22T02:34:42.039+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:42.039] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:43.047+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:43.047] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:44.039+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:44.039] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:45.040+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:45.040] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:46.040+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:46.040] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:47.041+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:47.041] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:48.041+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:48.041] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:49.042+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:49.042] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:50.043+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:50.043] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:51.044+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:51.044] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:51.596+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:34:51.596] clusterConfig unchanged [2021-10-22T02:34:52.044+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:52.044] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:53.044+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:53.044] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:54.044+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:54.044] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:55.051+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:55.051] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:56.045+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:56.045] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:57.046+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:57.046] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:58.046+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:58.046] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:34:59.046+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:34:59.046] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:00.046+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:00.046] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:01.050+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:01.050] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:01.738+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:35:01.738] clusterConfig unchanged [2021-10-22T02:35:02.063+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:02.063] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:03.047+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:03.047] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:04.047+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:04.047] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:05.047+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:05.047] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:06.047+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:06.047] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:07.048+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:07.048] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:08.048+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:08.048] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:09.049+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:09.049] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:10.050+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:10.050] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:11.051+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:11.051] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:11.840+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:35:11.840] clusterConfig unchanged [2021-10-22T02:35:12.051+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:12.051] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:13.052+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:13.052] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:14.067+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:14.067] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:15.055+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:15.055] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:16.052+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:16.052] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:17.053+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:17.053] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:18.053+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:18.053] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:19.054+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:19.054] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:20.055+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:20.055] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:20.256+0000] [.info] [src/director/director.go:computePlan:280] [02:35:20.256] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:35:20.256+0000] [.info] [src/director/director.go:tracef:794] [02:35:20.256] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:35:20.256+0000] [.info] [src/director/director.go:tracef:794] [02:35:20.256] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:35:20.256+0000] [.info] [src/director/director.go:tracef:794] [02:35:20.256] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:35:20.256+0000] [.info] [src/director/director.go:tracef:794] [02:35:20.256] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:35:20.256+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:35:20.256] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:35:20.413+0000] [.info] [src/director/director.go:tracef:794] [02:35:20.413] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:35:20.413+0000] [.info] [src/director/director.go:tracef:794] [02:35:20.413] because [2021-10-22T02:35:20.413+0000] [.info] [src/director/director.go:tracef:794] [02:35:20.413] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:35:20.413+0000] [.info] [src/director/director.go:tracef:794] [02:35:20.413] because ['currentState.CanBounceStop' = true] [2021-10-22T02:35:20.580+0000] [.info] [src/director/director.go:tracef:794] [02:35:20.580] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:35:20.580+0000] [.info] [src/director/director.go:tracef:794] [02:35:20.580] because [2021-10-22T02:35:20.581+0000] [.info] [src/director/director.go:tracef:794] [02:35:20.581] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:35:20.581+0000] [.info] [src/director/director.go:tracef:794] [02:35:20.581] because ['currentState.Up' = false] [2021-10-22T02:35:20.581+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:35:20.581] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:35:20.581+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:35:20.581] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:35:20.663+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:35:20.663] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:35:20.663+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:35:20.663] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:35:20.671+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:35:20.671] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:35:20.673+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:35:20.673] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:35:20.673+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:35:20.673] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:35:20.673+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:35:20.673] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:35:20.674+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:35:20.674] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:35:20.894+0000] [.info] [src/director/director.go:tracef:794] [02:35:20.894] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:35:20.894+0000] [.info] [src/director/director.go:tracef:794] [02:35:20.894] because ['currentState.Up' = false] [2021-10-22T02:35:21.057+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:21.057] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:21.111+0000] [.info] [src/director/director.go:tracef:794] [02:35:21.111] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:35:21.111+0000] [.info] [src/director/director.go:tracef:794] [02:35:21.111] because ['currentState.Up' = false] [2021-10-22T02:35:21.111+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:35:21.111] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:35:21.349+0000] [.info] [src/director/director.go:tracef:794] [02:35:21.349] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:35:21.349+0000] [.info] [src/director/director.go:tracef:794] [02:35:21.349] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:35:21.571+0000] [.info] [src/director/director.go:tracef:794] [02:35:21.571] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:35:21.571+0000] [.info] [src/director/director.go:tracef:794] [02:35:21.571] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:35:21.571+0000] [.info] [src/director/director.go:tracef:794] [02:35:21.571] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:35:21.571+0000] [.info] [src/director/director.go:tracef:794] [02:35:21.571] because ['currentState.Up' = false] [2021-10-22T02:35:21.571+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:35:21.571] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:35:21.571+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:35:21.571] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:35:21.607+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:35:21.607] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:35:21.608+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:35:21.607] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49588 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:35:22.011+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:35:22.011] clusterConfig unchanged [2021-10-22T02:35:22.066+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:22.066] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:22.382+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:35:22.382] ...Finished start [pid=36557] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49588,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:35:22.546+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:35:22.546] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 164.211229ms. [2021-10-22T02:35:22.546+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:35:22.546] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49588 (local=false) [2021-10-22T02:35:22.546+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:35:22.546] Dropping existing documents from system.replset [2021-10-22T02:35:22.739+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:35:22.739] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:35:22.744+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:35:22.744] Inserting oplogTruncateAfterPoint document [2021-10-22T02:35:22.744+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:35:22.744] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:35:22.745+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:35:22.745] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:35:22.745+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:35:22.745] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:35:22.757+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:35:22.757] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:35:22.757+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:35:22.757] Setting the minvalid document to [{_id ObjectID("6172236a523370002becc8ec")} {t -1} {ts {0 1}}] [2021-10-22T02:35:22.769+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:35:22.769] Inserted [{_id ObjectID("6172236a523370002becc8ec")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:35:22.769+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:35:22.769] Inserting new replica set configuration document. [2021-10-22T02:35:22.769+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:35:22.769] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:35:22.782+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:35:22.782] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:35:22.786+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:35:22.786] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:35:22.786+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:35:22.786] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:35:22.786+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:35:22.786] Initiating stopEphemeralMongo [2021-10-22T02:35:22.786+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:35:22.786] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49588 (local=false) [2021-10-22T02:35:22.786+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:35:22.786] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49588 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:35:23.066+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:23.066] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:23.124+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:35:23.124] clusterConfig unchanged [2021-10-22T02:35:23.243+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:35:23.243] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 4.405176ms. [2021-10-22T02:35:23.243+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:35:23.243] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:49588 (local=false) [2021-10-22T02:35:23.244+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:35:23.244] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 38.819µs. [2021-10-22T02:35:23.244+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:35:23.244] Successfully finished running stopEphemeralMongo [2021-10-22T02:35:23.244+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:35:23.244] Successfully stopped existing ephemeral mongod. [2021-10-22T02:35:23.244+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:35:23.244] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:35:23.244+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:35:23.244] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:35:23.289+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:35:23.289] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:35:23.290+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:35:23.290] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59752 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:35:24.066+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:24.066] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:24.257+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:35:24.257] clusterConfig unchanged [2021-10-22T02:35:24.506+0000] [.error] [src/action/start.go:func1:93] [103] [02:35:24.506] Error starting mongod : [02:35:24.506] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59752,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59752 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:35:23.322Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 38023 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:35:23.343+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:35:23.343+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:35:23.343+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59752,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:35:23.344+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:35:23.346+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:35:23.346+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:35:23.369+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:35:23.369+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:35:23.369+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:35:23.415+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:35:23.415+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:35:23.967+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870123:967042][38023:0x7fd02aa08bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:35:24.077+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870124:77732][38023:0x7fd02aa08bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:35:24.188+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870124:188633][38023:0x7fd02aa08bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35456 to 2/256"}} {"t":{"$date":"2021-10-22T02:35:24.313+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870124:313550][38023:0x7fd02aa08bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:35:24.392+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870124:392920][38023:0x7fd02aa08bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:35:24.455+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870124:455335][38023:0x7fd02aa08bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:35:24.455+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870124:455401][38023:0x7fd02aa08bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:35:24.506+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:35:24.506] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59752 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:35:24.506] Error starting mongod : [02:35:24.506] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59752,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59752 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:35:23.322Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 38023 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:35:23.343+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:35:23.343+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:35:23.343+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59752,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:35:23.344+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:35:23.346+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:35:23.346+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:35:23.369+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:35:23.369+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:35:23.369+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:35:23.415+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:35:23.415+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:35:23.967+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870123:967042][38023:0x7fd02aa08bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:35:24.077+00:00"},"s":"I"...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:35:24.506+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:35:24.506] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:35:24.506] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59752 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:35:24.506] Error starting mongod : [02:35:24.506] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59752,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59752 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:35:23.322Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 38023 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:35:23.343+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:35:23.343+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:35:23.343+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59752,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:35:23.344+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:35:23.346+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:35:23.346+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:35:23.369+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:35:23.369+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:35:23.369+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:35:23.415+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:35:23.415+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:35:23.967+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870123:967042][38023:0x7fd02aa08bc0], tx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:35:24.506+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:35:24.506] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:35:24.507+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:35:24.507] Initiating stopEphemeralMongo [2021-10-22T02:35:24.507+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:35:24.507] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:35:24.507+0000] [.error] [src/director/director.go:executePlan:962] [02:35:24.507] Failed to apply action. Result = : [02:35:24.506] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:35:24.506] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59752 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:35:24.506] Error starting mongod : [02:35:24.506] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59752,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59752 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:35:23.322Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 38023 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:35:23.343+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:35:23.343+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:35:23.343+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59752,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:35:23.344+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:35:23.346+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:35:23.346+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:35:23.369+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:35:23.369+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:35:23.369+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:35:23.415+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:35:23.415+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:35:23.967+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:35:24.507+0000] [.error] [src/director/director.go:planAndExecute:567] [02:35:24.507] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:35:24.507] Failed to apply action. Result = : [02:35:24.506] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:35:24.506] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59752 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:35:24.506] Error starting mongod : [02:35:24.506] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59752,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59752 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:35:23.322Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 38023 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:35:23.343+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:35:23.343+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:35:23.343+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59752,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:35:23.344+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:35:23.346+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:35:23.346+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:35:23.369+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:35:23.369+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:35:23.369+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:35:23.415+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:35:23.415+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:35:24.507+0000] [.error] [src/director/director.go:mainLoop:399] [02:35:24.507] Failed to planAndExecute : [02:35:24.507] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:35:24.507] Failed to apply action. Result = : [02:35:24.506] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:35:24.506] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:59752 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:35:24.506] Error starting mongod : [02:35:24.506] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59752,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 59752 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:35:23.322Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 38023 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:35:23.343+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:35:23.343+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:35:23.343+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":59752,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:35:23.344+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:35:23.346+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:35:23.346+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:35:23.369+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:35:23.369+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:35:23.369+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:35:23.415+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:35:23.415+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:35:25.065+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:25.065] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:25.497+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:35:25.497] clusterConfig unchanged [2021-10-22T02:35:26.063+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:26.063] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:27.063+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:27.063] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:28.070+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:28.070] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:29.080+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:29.080] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:30.072+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:30.072] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:31.072+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:31.072] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:32.070+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:32.070] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:33.073+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:33.073] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:33.758+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:35:33.758] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 13.08313895s [2021-10-22T02:35:34.071+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:34.071] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:35.071+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:35.071] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:35.589+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:35:35.589] clusterConfig unchanged [2021-10-22T02:35:36.074+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:36.074] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:37.072+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:37.072] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:38.072+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:38.072] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:39.073+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:39.073] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:40.073+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:40.073] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:41.074+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:41.074] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:41.412+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:35:41.412] memory:rss=56377344 alloc=9056904 totalalloc=11680266320 mallocs=149166429 frees=149103057 heapinuse=17506304 heapobjects=63372 gcpausetotalns=1653611813 gcpauselastns=144527 gcnum=1695 utimens=259083056000 stimens=104385887000 minflt=632026 majflt=405 [2021-10-22T02:35:42.074+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:42.074] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:43.075+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:43.075] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:44.076+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:44.076] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:44.463+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [02:35:44.463] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:59752 (local=false) because it is not up [2021-10-22T02:35:45.077+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:45.077] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:45.691+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:35:45.691] clusterConfig unchanged [2021-10-22T02:35:46.078+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:46.078] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:47.078+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:47.078] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:48.082+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:48.082] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:49.079+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:49.079] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:50.080+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:50.080] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:51.081+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:51.081] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:52.081+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:52.081] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:53.081+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:53.081] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:54.082+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:54.082] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:55.084+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:55.084] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:55.794+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:35:55.794] clusterConfig unchanged [2021-10-22T02:35:56.084+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:56.084] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:57.085+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:57.085] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:58.086+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:58.086] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:35:59.086+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:35:59.086] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:00.088+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:00.087] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:01.088+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:01.088] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:02.094+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:02.094] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:03.089+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:03.089] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:04.090+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:04.089] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:05.090+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:05.090] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:05.927+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:36:05.926] clusterConfig unchanged [2021-10-22T02:36:06.090+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:06.090] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:07.091+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:07.091] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:08.092+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:08.092] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:09.092+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:09.092] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:10.092+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:10.092] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:11.109+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:11.109] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:12.094+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:12.094] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:13.094+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:13.094] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:14.095+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:14.095] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:15.095+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:15.095] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:16.027+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:36:16.027] clusterConfig unchanged [2021-10-22T02:36:16.096+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:16.096] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:17.097+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:17.097] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:18.097+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:18.097] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:18.567+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:36:18.567] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:36:18.569+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:18.569] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:36:18.569+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [02:36:18.569] Could not get version : [02:36:18.569] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:36:18.569+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [02:36:18.569] Error getting version for log lines. Skipping : [02:36:18.569] Could not get version : [02:36:18.569] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:36:19.097+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:19.097] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:20.098+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:20.098] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:21.099+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:21.099] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:22.099+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:22.099] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:23.100+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:23.100] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:24.101+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:24.101] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:24.674+0000] [.info] [src/director/director.go:computePlan:280] [02:36:24.674] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:36:24.674+0000] [.info] [src/director/director.go:tracef:794] [02:36:24.674] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:36:24.674+0000] [.info] [src/director/director.go:tracef:794] [02:36:24.674] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:36:24.674+0000] [.info] [src/director/director.go:tracef:794] [02:36:24.674] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:36:24.674+0000] [.info] [src/director/director.go:tracef:794] [02:36:24.674] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:36:24.674+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:36:24.674] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:36:24.836+0000] [.info] [src/director/director.go:tracef:794] [02:36:24.836] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:36:24.836+0000] [.info] [src/director/director.go:tracef:794] [02:36:24.836] because [2021-10-22T02:36:24.836+0000] [.info] [src/director/director.go:tracef:794] [02:36:24.836] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:36:24.836+0000] [.info] [src/director/director.go:tracef:794] [02:36:24.836] because ['currentState.CanBounceStop' = true] [2021-10-22T02:36:24.988+0000] [.info] [src/director/director.go:tracef:794] [02:36:24.988] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:36:24.989+0000] [.info] [src/director/director.go:tracef:794] [02:36:24.989] because [2021-10-22T02:36:24.989+0000] [.info] [src/director/director.go:tracef:794] [02:36:24.989] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:36:24.989+0000] [.info] [src/director/director.go:tracef:794] [02:36:24.989] because ['currentState.Up' = false] [2021-10-22T02:36:24.989+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:36:24.989] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:36:24.989+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:36:24.989] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:36:25.067+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:36:25.067] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:36:25.076+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:36:25.076] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:36:25.076+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:36:25.076] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:36:25.077+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:36:25.077] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:36:25.078+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:36:25.078] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:36:25.078+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:36:25.078] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:36:25.078+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:36:25.078] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:36:25.080+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:36:25.080] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:36:25.108+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:25.108] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:25.336+0000] [.info] [src/director/director.go:tracef:794] [02:36:25.336] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:36:25.336+0000] [.info] [src/director/director.go:tracef:794] [02:36:25.336] because ['currentState.Up' = false] [2021-10-22T02:36:25.550+0000] [.info] [src/director/director.go:tracef:794] [02:36:25.550] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:36:25.550+0000] [.info] [src/director/director.go:tracef:794] [02:36:25.550] because ['currentState.Up' = false] [2021-10-22T02:36:25.550+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:36:25.550] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:36:25.766+0000] [.info] [src/director/director.go:tracef:794] [02:36:25.766] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:36:25.766+0000] [.info] [src/director/director.go:tracef:794] [02:36:25.766] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:36:26.009+0000] [.info] [src/director/director.go:tracef:794] [02:36:26.009] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:36:26.009+0000] [.info] [src/director/director.go:tracef:794] [02:36:26.009] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:36:26.009+0000] [.info] [src/director/director.go:tracef:794] [02:36:26.009] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:36:26.009+0000] [.info] [src/director/director.go:tracef:794] [02:36:26.009] because ['currentState.Up' = false] [2021-10-22T02:36:26.009+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:36:26.009] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:36:26.009+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:36:26.009] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:36:26.045+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:36:26.045] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:36:26.046+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:36:26.046] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 47253 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:36:26.107+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:26.107] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:26.237+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:36:26.237] clusterConfig unchanged [2021-10-22T02:36:26.792+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:36:26.792] ...Finished start [pid=57330] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":47253,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:36:26.920+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:36:26.920] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 127.507899ms. [2021-10-22T02:36:26.920+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:36:26.920] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47253 (local=false) [2021-10-22T02:36:26.920+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:36:26.920] Dropping existing documents from system.replset [2021-10-22T02:36:27.069+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:36:27.069] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:36:27.075+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:36:27.075] Inserting oplogTruncateAfterPoint document [2021-10-22T02:36:27.075+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:36:27.075] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:36:27.077+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:36:27.077] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:36:27.077+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:36:27.077] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:36:27.090+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:36:27.090] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:36:27.090+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:36:27.090] Setting the minvalid document to [{_id ObjectID("617223ab523370002becc8fa")} {t -1} {ts {0 1}}] [2021-10-22T02:36:27.105+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:36:27.105] Inserted [{_id ObjectID("617223ab523370002becc8fa")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:36:27.105+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:36:27.105] Inserting new replica set configuration document. [2021-10-22T02:36:27.105+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:36:27.105] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:36:27.106+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:27.106] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:27.119+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:36:27.119] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:36:27.141+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:36:27.141] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:36:27.141+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:36:27.141] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:36:27.141+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:36:27.141] Initiating stopEphemeralMongo [2021-10-22T02:36:27.141+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:36:27.141] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47253 (local=false) [2021-10-22T02:36:27.141+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:36:27.141] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47253 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:36:27.350+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:36:27.350] clusterConfig unchanged [2021-10-22T02:36:27.652+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:36:27.652] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.131026ms. [2021-10-22T02:36:27.652+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:36:27.652] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:47253 (local=false) [2021-10-22T02:36:27.652+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:36:27.652] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.243µs. [2021-10-22T02:36:27.652+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:36:27.652] Successfully finished running stopEphemeralMongo [2021-10-22T02:36:27.652+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:36:27.652] Successfully stopped existing ephemeral mongod. [2021-10-22T02:36:27.652+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:36:27.652] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:36:27.652+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:36:27.652] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:36:27.687+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:36:27.687] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:36:27.687+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:36:27.687] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35368 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:36:28.108+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:28.108] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:28.474+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:36:28.474] clusterConfig unchanged [2021-10-22T02:36:28.899+0000] [.error] [src/action/start.go:func1:93] [103] [02:36:28.899] Error starting mongod : [02:36:28.899] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35368,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35368 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 58875 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:36:27.762+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:36:27.762+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:36:27.762+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35368,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:36:27.764+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:36:27.766+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:36:27.768+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:36:27.819+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:36:27.819+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:36:27.830+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:36:27.877+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:36:27.877+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:36:28.375+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870188:375885][58875:0x7fed87fdfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:36:28.483+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870188:483546][58875:0x7fed87fdfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:36:28.592+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870188:592014][58875:0x7fed87fdfbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T02:36:28.711+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870188:711783][58875:0x7fed87fdfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:36:28.785+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870188:785971][58875:0x7fed87fdfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:36:28.850+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870188:850124][58875:0x7fed87fdfbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:36:28.850+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870188:850200][58875:0x7fed87fdfbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:36:28.852+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:36:28.899+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:36:28.899] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35368 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:36:28.899] Error starting mongod : [02:36:28.899] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35368,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35368 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 58875 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:36:27.762+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:36:27.762+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:36:27.762+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35368,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:36:27.764+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:36:27.766+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:36:27.768+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:36:27.819+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:36:27.819+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:36:27.830+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:36:27.877+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:36:27.877+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:36:28.375+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870188:375885][58875:0x7fed87fdfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:36:28.483+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870188:483546][58875:0x7fed87fdfbc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:36:28.900+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:36:28.900] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:36:28.899] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35368 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:36:28.899] Error starting mongod : [02:36:28.899] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35368,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35368 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 58875 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:36:27.762+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:36:27.762+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:36:27.762+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35368,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:36:27.764+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:36:27.766+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:36:27.768+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:36:27.819+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:36:27.819+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:36:27.830+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:36:27.877+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:36:27.877+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:36:28.375+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870188:375885][58875:0x7fed87fdfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:36:28.483+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:36:28.900+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:36:28.900] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:36:28.900+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:36:28.900] Initiating stopEphemeralMongo [2021-10-22T02:36:28.900+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:36:28.900] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:36:28.900+0000] [.error] [src/director/director.go:executePlan:962] [02:36:28.900] Failed to apply action. Result = : [02:36:28.900] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:36:28.899] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35368 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:36:28.899] Error starting mongod : [02:36:28.899] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35368,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35368 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 58875 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:36:27.762+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:36:27.762+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:36:27.762+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35368,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:36:27.764+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:36:27.766+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:36:27.768+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:36:27.819+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:36:27.819+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:36:27.830+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:36:27.877+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:36:27.877+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:36:28.375+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870188:375885][58875:0x7fed87fdfbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:36:28.901+0000] [.error] [src/director/director.go:planAndExecute:567] [02:36:28.901] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:36:28.900] Failed to apply action. Result = : [02:36:28.900] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:36:28.899] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35368 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:36:28.899] Error starting mongod : [02:36:28.899] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35368,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35368 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 58875 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:36:27.762+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:36:27.762+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:36:27.762+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35368,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:36:27.764+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:36:27.766+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:36:27.768+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:36:27.819+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:36:27.819+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:36:27.830+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:36:27.877+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:36:27.877+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:36:28.375+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:36:28.901+0000] [.error] [src/director/director.go:mainLoop:399] [02:36:28.901] Failed to planAndExecute : [02:36:28.901] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:36:28.900] Failed to apply action. Result = : [02:36:28.900] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:36:28.899] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35368 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:36:28.899] Error starting mongod : [02:36:28.899] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35368,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35368 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 58875 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:36:27.762+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:36:27.762+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:36:27.762+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35368,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:36:27.764+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:36:27.766+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:36:27.768+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:36:27.819+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:36:27.819+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:36:27.830+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:36:27.877+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:36:27.877+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:36:28.375+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:36:29.107+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:29.107] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:29.733+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:36:29.733] clusterConfig unchanged [2021-10-22T02:36:30.110+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:30.110] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:31.112+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:31.112] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:32.113+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:32.113] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:33.114+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:33.114] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:34.114+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:34.114] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:35.117+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:35.117] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:36.117+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:36.117] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:37.118+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:37.118] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:38.125+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:38.125] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:38.151+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:36:38.151] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 13.071076344s [2021-10-22T02:36:39.124+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:39.124] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:39.826+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:36:39.826] clusterConfig unchanged [2021-10-22T02:36:40.125+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:40.125] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:41.125+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:41.125] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:41.413+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:36:41.413] memory:rss=56700928 alloc=9198976 totalalloc=11774507392 mallocs=150372054 frees=150304823 heapinuse=17522688 heapobjects=67231 gcpausetotalns=1654849401 gcpauselastns=113872 gcnum=1707 utimens=261743962000 stimens=105566295000 minflt=638362 majflt=405 [2021-10-22T02:36:42.125+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:42.125] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:43.126+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:43.126] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:44.127+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:44.127] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:45.127+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:45.127] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:46.128+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:46.128] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:47.129+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:47.129] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:48.130+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:48.130] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:49.130+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:49.130] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:49.939+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:36:49.939] clusterConfig unchanged [2021-10-22T02:36:50.138+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:50.138] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:51.131+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:51.131] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:52.132+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:52.132] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:53.133+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:53.133] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:54.133+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:54.133] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:55.133+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:55.133] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:56.134+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:56.134] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:57.135+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:57.135] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:58.135+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:58.135] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:36:59.135+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:36:59.135] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:00.071+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:37:00.071] clusterConfig unchanged [2021-10-22T02:37:00.136+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:00.136] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:01.136+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:01.136] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:02.145+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:02.145] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:03.138+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:03.138] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:04.138+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:04.138] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:05.139+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:05.139] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:06.139+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:06.139] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:07.140+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:07.140] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:08.141+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:08.141] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:09.141+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:09.141] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:10.142+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:10.142] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:10.182+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:37:10.182] clusterConfig unchanged [2021-10-22T02:37:11.143+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:11.143] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:12.143+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:12.143] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:13.157+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:13.157] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:14.144+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:14.144] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:15.144+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:15.144] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:16.145+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:16.145] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:17.145+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:17.145] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:18.146+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:18.146] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:19.146+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:19.146] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:20.147+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:20.147] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:20.274+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:37:20.274] clusterConfig unchanged [2021-10-22T02:37:21.147+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:21.147] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:22.147+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:22.147] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:23.148+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:23.148] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:24.149+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:24.149] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:25.150+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:25.150] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:26.150+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:26.150] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:27.150+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:27.150] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:27.573+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:37:27.573] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:37:27.573+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:37:27.573] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:37:28.151+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:28.151] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:29.068+0000] [.info] [src/director/director.go:computePlan:280] [02:37:29.068] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:37:29.068+0000] [.info] [src/director/director.go:tracef:794] [02:37:29.068] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:37:29.068+0000] [.info] [src/director/director.go:tracef:794] [02:37:29.068] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:37:29.068+0000] [.info] [src/director/director.go:tracef:794] [02:37:29.068] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:37:29.068+0000] [.info] [src/director/director.go:tracef:794] [02:37:29.068] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:37:29.068+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:37:29.068] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:37:29.172+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:29.172] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:29.225+0000] [.info] [src/director/director.go:tracef:794] [02:37:29.225] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:37:29.225+0000] [.info] [src/director/director.go:tracef:794] [02:37:29.225] because [2021-10-22T02:37:29.225+0000] [.info] [src/director/director.go:tracef:794] [02:37:29.225] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:37:29.225+0000] [.info] [src/director/director.go:tracef:794] [02:37:29.225] because ['currentState.CanBounceStop' = true] [2021-10-22T02:37:29.376+0000] [.info] [src/director/director.go:tracef:794] [02:37:29.375] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:37:29.376+0000] [.info] [src/director/director.go:tracef:794] [02:37:29.376] because [2021-10-22T02:37:29.376+0000] [.info] [src/director/director.go:tracef:794] [02:37:29.376] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:37:29.376+0000] [.info] [src/director/director.go:tracef:794] [02:37:29.376] because ['currentState.Up' = false] [2021-10-22T02:37:29.376+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:37:29.376] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:37:29.376+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:37:29.376] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:37:29.461+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:37:29.461] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:37:29.469+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:37:29.469] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:37:29.469+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:37:29.469] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:37:29.470+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:37:29.470] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:37:29.471+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:37:29.471] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:37:29.471+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:37:29.471] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:37:29.471+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:37:29.471] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:37:29.472+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:37:29.472] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:37:29.711+0000] [.info] [src/director/director.go:tracef:794] [02:37:29.711] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:37:29.711+0000] [.info] [src/director/director.go:tracef:794] [02:37:29.711] because ['currentState.Up' = false] [2021-10-22T02:37:29.951+0000] [.info] [src/director/director.go:tracef:794] [02:37:29.951] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:37:29.951+0000] [.info] [src/director/director.go:tracef:794] [02:37:29.951] because ['currentState.Up' = false] [2021-10-22T02:37:29.951+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:37:29.951] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:37:30.156+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:30.156] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:30.170+0000] [.info] [src/director/director.go:tracef:794] [02:37:30.170] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:37:30.171+0000] [.info] [src/director/director.go:tracef:794] [02:37:30.171] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:37:30.396+0000] [.info] [src/director/director.go:tracef:794] [02:37:30.396] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:37:30.396+0000] [.info] [src/director/director.go:tracef:794] [02:37:30.396] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:37:30.397+0000] [.info] [src/director/director.go:tracef:794] [02:37:30.397] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:37:30.397+0000] [.info] [src/director/director.go:tracef:794] [02:37:30.397] because ['currentState.Up' = false] [2021-10-22T02:37:30.397+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:37:30.397] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:37:30.397+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:37:30.397] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:37:30.428+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:37:30.428] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:37:30.429+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:37:30.429] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 51141 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:37:30.490+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:37:30.490] clusterConfig unchanged [2021-10-22T02:37:31.154+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:37:31.154] ...Finished start [pid=4059] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":51141,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:37:31.158+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:31.158] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:31.292+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:37:31.292] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 138.184531ms. [2021-10-22T02:37:31.292+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:37:31.292] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51141 (local=false) [2021-10-22T02:37:31.292+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:37:31.292] Dropping existing documents from system.replset [2021-10-22T02:37:31.455+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:37:31.455] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:37:31.458+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:37:31.458] Inserting oplogTruncateAfterPoint document [2021-10-22T02:37:31.458+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:37:31.458] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:37:31.460+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:37:31.460] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:37:31.460+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:37:31.460] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:37:31.474+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:37:31.474] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:37:31.474+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:37:31.474] Setting the minvalid document to [{_id ObjectID("617223eb523370002becc908")} {t -1} {ts {0 1}}] [2021-10-22T02:37:31.486+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:37:31.486] Inserted [{_id ObjectID("617223eb523370002becc908")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:37:31.486+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:37:31.486] Inserting new replica set configuration document. [2021-10-22T02:37:31.486+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:37:31.486] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:37:31.500+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:37:31.500] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:37:31.503+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:37:31.503] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:37:31.503+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:37:31.503] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:37:31.503+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:37:31.503] Initiating stopEphemeralMongo [2021-10-22T02:37:31.503+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:37:31.503] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51141 (local=false) [2021-10-22T02:37:31.503+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:37:31.503] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51141 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:37:31.627+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:37:31.627] clusterConfig unchanged [2021-10-22T02:37:31.939+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:37:31.939] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.027567ms. [2021-10-22T02:37:31.939+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:37:31.939] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:51141 (local=false) [2021-10-22T02:37:31.939+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:37:31.939] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.075µs. [2021-10-22T02:37:31.939+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:37:31.939] Successfully finished running stopEphemeralMongo [2021-10-22T02:37:31.939+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:37:31.939] Successfully stopped existing ephemeral mongod. [2021-10-22T02:37:31.939+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:37:31.939] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:37:31.939+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:37:31.939] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:37:31.971+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:37:31.971] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:37:31.972+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:37:31.972] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49303 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:37:32.164+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:32.164] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:32.771+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:37:32.771] clusterConfig unchanged [2021-10-22T02:37:33.169+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:33.169] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:33.267+0000] [.error] [src/action/start.go:func1:93] [103] [02:37:33.267] Error starting mongod : [02:37:33.266] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49303,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49303 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 5534 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: nfig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:37:32.020+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:37:32.020+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:37:32.020+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49303,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:37:32.021+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:37:32.023+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:37:32.023+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:37:32.054+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:37:32.054+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:37:32.054+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:37:32.099+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:37:32.099+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:37:32.638+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870252:638744][5534:0x7f60c290ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:37:32.779+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870252:779490][5534:0x7f60c290ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:37:32.888+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870252:888561][5534:0x7f60c290ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T02:37:33.010+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870253:10086][5534:0x7f60c290ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:37:33.090+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870253:90412][5534:0x7f60c290ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:37:33.186+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870253:186971][5534:0x7f60c290ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:37:33.187+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870253:187045][5534:0x7f60c290ebc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:37:33.188+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:37:33.267+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:37:33.267] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49303 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:37:33.267] Error starting mongod : [02:37:33.266] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49303,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49303 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 5534 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: nfig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:37:32.020+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:37:32.020+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:37:32.020+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49303,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:37:32.021+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:37:32.023+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:37:32.023+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:37:32.054+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:37:32.054+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:37:32.054+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:37:32.099+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:37:32.099+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:37:32.638+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870252:638744][5534:0x7f60c290ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:37:32.779+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870252:779490][5534:0x7f60c290ebc0], txn-recov...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:37:33.267+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:37:33.267] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:37:33.267] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49303 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:37:33.267] Error starting mongod : [02:37:33.266] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49303,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49303 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 5534 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: nfig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:37:32.020+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:37:32.020+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:37:32.020+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49303,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:37:32.021+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:37:32.023+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:37:32.023+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:37:32.054+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:37:32.054+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:37:32.054+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:37:32.099+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:37:32.099+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:37:32.638+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870252:638744][5534:0x7f60c290ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:37:32.779+00:00"},"s":"I", "c":"STORAGE", "id":22430... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:37:33.267+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:37:33.267] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:37:33.267+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:37:33.267] Initiating stopEphemeralMongo [2021-10-22T02:37:33.267+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:37:33.267] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:37:33.268+0000] [.error] [src/director/director.go:executePlan:962] [02:37:33.268] Failed to apply action. Result = : [02:37:33.267] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:37:33.267] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49303 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:37:33.267] Error starting mongod : [02:37:33.266] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49303,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49303 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 5534 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: nfig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:37:32.020+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:37:32.020+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:37:32.020+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49303,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:37:32.021+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:37:32.023+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:37:32.023+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:37:32.054+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:37:32.054+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:37:32.054+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:37:32.099+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:37:32.099+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:37:32.638+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870252:638744][5534:0x7f60c290ebc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:37:33.268+0000] [.error] [src/director/director.go:planAndExecute:567] [02:37:33.268] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:37:33.268] Failed to apply action. Result = : [02:37:33.267] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:37:33.267] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49303 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:37:33.267] Error starting mongod : [02:37:33.266] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49303,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49303 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 5534 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: nfig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:37:32.020+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:37:32.020+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:37:32.020+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49303,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:37:32.021+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:37:32.023+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:37:32.023+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:37:32.054+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:37:32.054+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:37:32.054+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:37:32.099+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:37:32.099+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:37:32.638+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:37:33.268+0000] [.error] [src/director/director.go:mainLoop:399] [02:37:33.268] Failed to planAndExecute : [02:37:33.268] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:37:33.268] Failed to apply action. Result = : [02:37:33.267] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:37:33.267] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:49303 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:37:33.267] Error starting mongod : [02:37:33.266] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49303,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 49303 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 5534 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: nfig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:37:32.020+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:37:32.020+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:37:32.020+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":49303,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:37:32.021+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:37:32.023+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:37:32.023+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:37:32.054+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:37:32.054+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:37:32.054+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:37:32.099+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:37:32.099+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:37:32.638+00:00"},"s":"I... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:37:34.039+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:37:34.039] clusterConfig unchanged [2021-10-22T02:37:34.169+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:34.169] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:35.171+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:35.171] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:36.171+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:36.171] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:37.172+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:37.172] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:38.172+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:38.172] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:39.175+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [02:37:39.175] lsblk cache invalidated, running lsblk command [2021-10-22T02:37:39.177+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:39.177] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:40.176+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:40.176] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:41.175+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:41.175] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:41.414+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:37:41.414] memory:rss=59445248 alloc=16060800 totalalloc=11866190400 mallocs=151566232 frees=151423757 heapinuse=19685376 heapobjects=142475 gcpausetotalns=1655918352 gcpauselastns=78371 gcnum=1718 utimens=264158377000 stimens=106608995000 minflt=643462 majflt=405 [2021-10-22T02:37:42.183+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:42.183] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:42.304+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:37:42.304] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.831755619s [2021-10-22T02:37:43.174+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:43.174] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:44.160+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:37:44.160] clusterConfig unchanged [2021-10-22T02:37:44.175+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:44.175] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:45.175+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:45.175] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:46.176+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:46.176] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:47.177+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:47.177] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:48.177+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:48.177] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:49.179+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:49.179] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:50.179+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:50.179] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:51.179+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:51.179] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:52.180+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:52.180] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:53.180+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:53.180] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:54.181+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:54.181] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:54.253+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:37:54.253] clusterConfig unchanged [2021-10-22T02:37:55.182+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:55.181] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:56.188+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:56.188] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:57.182+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:57.182] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:58.183+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:58.183] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:37:59.183+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:37:59.183] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:00.184+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:00.184] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:01.185+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:01.185] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:02.190+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:02.190] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:03.185+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:03.185] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:04.186+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:04.186] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:04.374+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:38:04.374] clusterConfig unchanged [2021-10-22T02:38:05.194+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:05.194] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:06.187+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:06.187] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:07.189+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:07.188] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:08.188+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:08.188] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:09.188+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:09.188] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:10.189+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:10.189] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:11.189+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:11.189] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:12.189+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:12.189] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:13.190+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:13.190] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:14.190+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:14.190] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:14.506+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:38:14.506] clusterConfig unchanged [2021-10-22T02:38:15.190+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:15.190] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:16.195+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:16.195] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:17.192+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:17.192] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:18.192+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:18.192] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:19.193+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:19.192] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:20.193+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:20.193] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:21.195+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:21.195] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:22.195+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:22.195] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:23.196+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:23.196] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:24.197+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:24.197] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:24.598+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:38:24.598] clusterConfig unchanged [2021-10-22T02:38:25.198+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:25.198] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:26.213+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:26.213] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:27.207+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:27.207] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:28.199+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:28.199] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:29.200+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:29.200] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:30.200+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:30.200] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:31.201+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:31.201] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:32.201+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:32.201] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:33.202+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:33.202] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:33.440+0000] [.info] [src/director/director.go:computePlan:280] [02:38:33.440] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:38:33.441+0000] [.info] [src/director/director.go:tracef:794] [02:38:33.441] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:38:33.441+0000] [.info] [src/director/director.go:tracef:794] [02:38:33.441] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:38:33.441+0000] [.info] [src/director/director.go:tracef:794] [02:38:33.441] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:38:33.441+0000] [.info] [src/director/director.go:tracef:794] [02:38:33.441] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:38:33.441+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:38:33.441] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:38:33.615+0000] [.info] [src/director/director.go:tracef:794] [02:38:33.615] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:38:33.615+0000] [.info] [src/director/director.go:tracef:794] [02:38:33.615] because [2021-10-22T02:38:33.615+0000] [.info] [src/director/director.go:tracef:794] [02:38:33.615] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:38:33.615+0000] [.info] [src/director/director.go:tracef:794] [02:38:33.615] because ['currentState.CanBounceStop' = true] [2021-10-22T02:38:33.771+0000] [.info] [src/director/director.go:tracef:794] [02:38:33.771] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:38:33.771+0000] [.info] [src/director/director.go:tracef:794] [02:38:33.771] because [2021-10-22T02:38:33.771+0000] [.info] [src/director/director.go:tracef:794] [02:38:33.771] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:38:33.771+0000] [.info] [src/director/director.go:tracef:794] [02:38:33.771] because ['currentState.Up' = false] [2021-10-22T02:38:33.772+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:38:33.772] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:38:33.772+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:38:33.772] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:38:34.120+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:38:34.120] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:38:34.128+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:38:34.128] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:38:34.128+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:38:34.128] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:38:34.130+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:38:34.130] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:38:34.130+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:38:34.130] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:38:34.130+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:38:34.130] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:38:34.131+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:38:34.131] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:38:34.207+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:34.207] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:34.380+0000] [.info] [src/director/director.go:tracef:794] [02:38:34.380] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:38:34.381+0000] [.info] [src/director/director.go:tracef:794] [02:38:34.381] because ['currentState.Up' = false] [2021-10-22T02:38:34.635+0000] [.info] [src/director/director.go:tracef:794] [02:38:34.635] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:38:34.635+0000] [.info] [src/director/director.go:tracef:794] [02:38:34.635] because ['currentState.Up' = false] [2021-10-22T02:38:34.635+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:38:34.635] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:38:34.790+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:38:34.790] clusterConfig unchanged [2021-10-22T02:38:34.851+0000] [.info] [src/director/director.go:tracef:794] [02:38:34.851] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:38:34.852+0000] [.info] [src/director/director.go:tracef:794] [02:38:34.851] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:38:35.063+0000] [.info] [src/director/director.go:tracef:794] [02:38:35.063] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:38:35.063+0000] [.info] [src/director/director.go:tracef:794] [02:38:35.063] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:38:35.063+0000] [.info] [src/director/director.go:tracef:794] [02:38:35.063] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:38:35.063+0000] [.info] [src/director/director.go:tracef:794] [02:38:35.063] because ['currentState.Up' = false] [2021-10-22T02:38:35.063+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:38:35.063] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:38:35.063+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:38:35.063] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:38:35.100+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:38:35.100] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:38:35.101+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:38:35.101] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 55159 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:38:35.209+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:35.209] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:35.848+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:38:35.848] ...Finished start [pid=24908] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":55159,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:38:35.971+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:38:35.970] clusterConfig unchanged [2021-10-22T02:38:35.972+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:38:35.972] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 124.613056ms. [2021-10-22T02:38:35.972+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:38:35.972] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:55159 (local=false) [2021-10-22T02:38:35.972+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:38:35.972] Dropping existing documents from system.replset [2021-10-22T02:38:36.126+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:38:36.126] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:38:36.133+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:38:36.133] Inserting oplogTruncateAfterPoint document [2021-10-22T02:38:36.133+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:38:36.133] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:38:36.134+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:38:36.134] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:38:36.134+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:38:36.134] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:38:36.147+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:38:36.147] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:38:36.147+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:38:36.147] Setting the minvalid document to [{_id ObjectID("6172242c523370002becc916")} {t -1} {ts {0 1}}] [2021-10-22T02:38:36.160+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:38:36.160] Inserted [{_id ObjectID("6172242c523370002becc916")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:38:36.160+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:38:36.160] Inserting new replica set configuration document. [2021-10-22T02:38:36.160+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:38:36.160] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:38:36.182+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:38:36.181] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:38:36.185+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:38:36.185] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:38:36.185+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:38:36.185] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:38:36.185+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:38:36.185] Initiating stopEphemeralMongo [2021-10-22T02:38:36.185+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:38:36.185] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:55159 (local=false) [2021-10-22T02:38:36.185+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:38:36.185] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:55159 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:38:36.217+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:36.217] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:36.527+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:38:36.527] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.072224ms. [2021-10-22T02:38:36.527+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:38:36.527] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:55159 (local=false) [2021-10-22T02:38:36.528+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:38:36.528] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.085µs. [2021-10-22T02:38:36.528+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:38:36.528] Successfully finished running stopEphemeralMongo [2021-10-22T02:38:36.528+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:38:36.528] Successfully stopped existing ephemeral mongod. [2021-10-22T02:38:36.528+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:38:36.528] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:38:36.528+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:38:36.528] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:38:36.561+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:38:36.561] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:38:36.562+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:38:36.562] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 54972 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:38:37.075+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:38:37.075] clusterConfig unchanged [2021-10-22T02:38:37.209+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:37.209] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:37.754+0000] [.error] [src/action/start.go:func1:93] [103] [02:38:37.754] Error starting mongod : [02:38:37.754] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54972,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 54972 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:38:36.593Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 26364 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:38:36.620+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:38:36.620+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:38:36.621+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54972,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:38:36.622+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:38:36.626+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:38:36.627+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:38:36.637+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:38:36.637+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:38:36.637+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:38:36.685+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:38:36.685+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:38:37.233+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870317:233733][26364:0x7f59f3cfdbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:38:37.348+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870317:348406][26364:0x7f59f3cfdbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:38:37.456+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870317:456445][26364:0x7f59f3cfdbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:38:37.574+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870317:574405][26364:0x7f59f3cfdbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:38:37.650+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870317:650156][26364:0x7f59f3cfdbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:38:37.707+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870317:707073][26364:0x7f59f3cfdbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:38:37.707+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870317:707131][26364:0x7f59f3cfdbc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:38:37.755+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:38:37.755] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:54972 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:38:37.754] Error starting mongod : [02:38:37.754] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54972,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 54972 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:38:36.593Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 26364 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:38:36.620+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:38:36.620+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:38:36.621+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54972,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:38:36.622+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:38:36.626+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:38:36.627+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:38:36.637+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:38:36.637+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:38:36.637+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:38:36.685+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:38:36.685+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:38:37.233+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870317:233733][26364:0x7f59f3cfdbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:38:37.348+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:38:37.755+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:38:37.755] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:38:37.755] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:54972 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:38:37.754] Error starting mongod : [02:38:37.754] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54972,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 54972 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:38:36.593Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 26364 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:38:36.620+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:38:36.620+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:38:36.621+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54972,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:38:36.622+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:38:36.626+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:38:36.627+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:38:36.637+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:38:36.637+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:38:36.637+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:38:36.685+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:38:36.685+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:38:37.233+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870317:233733][26364:0x7f59f3cfdbc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:38:37.755+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:38:37.755] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:38:37.755+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:38:37.755] Initiating stopEphemeralMongo [2021-10-22T02:38:37.755+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:38:37.755] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:38:37.756+0000] [.error] [src/director/director.go:executePlan:962] [02:38:37.755] Failed to apply action. Result = : [02:38:37.755] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:38:37.755] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:54972 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:38:37.754] Error starting mongod : [02:38:37.754] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54972,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 54972 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:38:36.593Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 26364 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:38:36.620+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:38:36.620+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:38:36.621+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54972,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:38:36.622+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:38:36.626+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:38:36.627+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:38:36.637+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:38:36.637+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:38:36.637+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:38:36.685+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:38:36.685+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:38:37.233+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:38:37.756+0000] [.error] [src/director/director.go:planAndExecute:567] [02:38:37.756] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:38:37.755] Failed to apply action. Result = : [02:38:37.755] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:38:37.755] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:54972 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:38:37.754] Error starting mongod : [02:38:37.754] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54972,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 54972 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:38:36.593Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 26364 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:38:36.620+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:38:36.620+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:38:36.621+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54972,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:38:36.622+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:38:36.626+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:38:36.627+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:38:36.637+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:38:36.637+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:38:36.637+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:38:36.685+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:38:36.685+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:38:37.756+0000] [.error] [src/director/director.go:mainLoop:399] [02:38:37.756] Failed to planAndExecute : [02:38:37.756] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:38:37.755] Failed to apply action. Result = : [02:38:37.755] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:38:37.755] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:54972 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:38:37.754] Error starting mongod : [02:38:37.754] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54972,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 54972 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:38:36.593Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 26364 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:38:36.620+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:38:36.620+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:38:36.621+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54972,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:38:36.622+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:38:36.626+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:38:36.627+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:38:36.637+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:38:36.637+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:38:36.637+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:38:36.685+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:38:36.685+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:38:38.217+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:38.217] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:38.304+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:38:38.304] clusterConfig unchanged [2021-10-22T02:38:39.211+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:39.211] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:40.210+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:40.210] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:41.211+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:41.211] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:41.415+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:38:41.415] memory:rss=59015168 alloc=13550920 totalalloc=11958256192 mallocs=152763011 frees=152657082 heapinuse=19243008 heapobjects=105929 gcpausetotalns=1666399017 gcpauselastns=1010598 gcnum=1730 utimens=266441909000 stimens=107675777000 minflt=648829 majflt=405 [2021-10-22T02:38:42.210+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:42.210] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:43.221+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:43.221] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:44.222+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:44.222] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:44.493+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [02:38:44.493] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:54972 (local=false) because it is not up [2021-10-22T02:38:45.223+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:45.223] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:46.224+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:46.224] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:47.029+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:38:47.029] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.897941688s [2021-10-22T02:38:47.222+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:47.222] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:48.223+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:48.223] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:48.427+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:38:48.427] clusterConfig unchanged [2021-10-22T02:38:49.224+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:49.224] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:50.224+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:50.224] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:51.224+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:51.224] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:52.225+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:52.225] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:53.226+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:53.226] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:54.226+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:54.226] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:55.228+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:55.228] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:56.228+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:56.228] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:57.228+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:57.228] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:58.229+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:58.229] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:38:58.539+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:38:58.539] clusterConfig unchanged [2021-10-22T02:38:59.230+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:38:59.230] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:00.243+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:00.243] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:01.234+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:01.234] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:02.231+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:02.231] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:03.232+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:03.232] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:04.232+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:04.232] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:05.233+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:05.233] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:06.233+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:06.233] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:07.242+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:07.242] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:08.235+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:08.235] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:08.680+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:39:08.680] clusterConfig unchanged [2021-10-22T02:39:09.235+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:09.235] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:10.236+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:10.236] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:11.236+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:11.236] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:12.237+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:12.237] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:13.238+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:13.238] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:14.238+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:14.238] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:15.238+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:15.238] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:16.239+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:16.239] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:17.240+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:17.240] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:18.240+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:18.240] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:18.804+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:39:18.804] clusterConfig unchanged [2021-10-22T02:39:19.241+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:19.241] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:20.241+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:20.241] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:21.242+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:21.242] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:22.243+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:22.243] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:23.244+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:23.244] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:24.245+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:24.245] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:25.246+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:25.246] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:26.247+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:26.246] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:27.247+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:27.247] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:28.248+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:28.248] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:28.934+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:39:28.934] clusterConfig unchanged [2021-10-22T02:39:29.255+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:29.255] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:30.248+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:30.248] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:31.249+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:31.249] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:32.250+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:32.250] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:33.252+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:33.252] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:34.251+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:34.251] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:35.251+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:35.251] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:36.252+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:36.252] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:37.252+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:37.252] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:37.923+0000] [.info] [src/director/director.go:computePlan:280] [02:39:37.923] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:39:37.923+0000] [.info] [src/director/director.go:tracef:794] [02:39:37.923] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:39:37.923+0000] [.info] [src/director/director.go:tracef:794] [02:39:37.923] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:39:37.924+0000] [.info] [src/director/director.go:tracef:794] [02:39:37.924] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:39:37.924+0000] [.info] [src/director/director.go:tracef:794] [02:39:37.924] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:39:37.924+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:39:37.924] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:39:38.089+0000] [.info] [src/director/director.go:tracef:794] [02:39:38.089] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:39:38.089+0000] [.info] [src/director/director.go:tracef:794] [02:39:38.089] because [2021-10-22T02:39:38.089+0000] [.info] [src/director/director.go:tracef:794] [02:39:38.089] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:39:38.089+0000] [.info] [src/director/director.go:tracef:794] [02:39:38.089] because ['currentState.CanBounceStop' = true] [2021-10-22T02:39:38.243+0000] [.info] [src/director/director.go:tracef:794] [02:39:38.243] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:39:38.243+0000] [.info] [src/director/director.go:tracef:794] [02:39:38.243] because [2021-10-22T02:39:38.243+0000] [.info] [src/director/director.go:tracef:794] [02:39:38.243] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:39:38.243+0000] [.info] [src/director/director.go:tracef:794] [02:39:38.243] because ['currentState.Up' = false] [2021-10-22T02:39:38.243+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:39:38.243] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:39:38.243+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:39:38.243] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:39:38.253+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:38.253] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:38.329+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:39:38.329] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:39:38.337+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:39:38.337] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:39:38.337+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:39:38.337] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:39:38.338+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:39:38.338] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:39:38.339+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:39:38.339] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:39:38.339+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:39:38.339] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:39:38.339+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:39:38.339] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:39:38.341+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:39:38.341] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:39:38.582+0000] [.info] [src/director/director.go:tracef:794] [02:39:38.582] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:39:38.582+0000] [.info] [src/director/director.go:tracef:794] [02:39:38.582] because ['currentState.Up' = false] [2021-10-22T02:39:38.779+0000] [.info] [src/director/director.go:tracef:794] [02:39:38.779] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:39:38.780+0000] [.info] [src/director/director.go:tracef:794] [02:39:38.779] because ['currentState.Up' = false] [2021-10-22T02:39:38.780+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:39:38.780] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:39:39.009+0000] [.info] [src/director/director.go:tracef:794] [02:39:39.009] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:39:39.009+0000] [.info] [src/director/director.go:tracef:794] [02:39:39.009] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:39:39.145+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:39:39.145] clusterConfig unchanged [2021-10-22T02:39:39.195+0000] [.info] [src/director/director.go:tracef:794] [02:39:39.195] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:39:39.195+0000] [.info] [src/director/director.go:tracef:794] [02:39:39.195] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:39:39.196+0000] [.info] [src/director/director.go:tracef:794] [02:39:39.196] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:39:39.196+0000] [.info] [src/director/director.go:tracef:794] [02:39:39.196] because ['currentState.Up' = false] [2021-10-22T02:39:39.196+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:39:39.196] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:39:39.196+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:39:39.196] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:39:39.227+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:39:39.227] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:39:39.228+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:39:39.228] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 44938 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:39:39.259+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:39.259] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:39.958+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:39:39.958] ...Finished start [pid=45543] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":44938,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:39:40.159+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:39:40.159] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 200.478515ms. [2021-10-22T02:39:40.159+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:39:40.159] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44938 (local=false) [2021-10-22T02:39:40.159+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:39:40.159] Dropping existing documents from system.replset [2021-10-22T02:39:40.263+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:40.263] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:40.277+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:39:40.277] clusterConfig unchanged [2021-10-22T02:39:40.314+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:39:40.314] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:39:40.318+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:39:40.318] Inserting oplogTruncateAfterPoint document [2021-10-22T02:39:40.318+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:39:40.318] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:39:40.319+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:39:40.319] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:39:40.319+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:39:40.319] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:39:40.333+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:39:40.332] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:39:40.333+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:39:40.333] Setting the minvalid document to [{_id ObjectID("6172246c523370002becc924")} {t -1} {ts {0 1}}] [2021-10-22T02:39:40.345+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:39:40.345] Inserted [{_id ObjectID("6172246c523370002becc924")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:39:40.345+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:39:40.345] Inserting new replica set configuration document. [2021-10-22T02:39:40.345+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:39:40.345] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:39:40.358+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:39:40.358] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:39:40.362+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:39:40.362] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:39:40.362+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:39:40.362] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:39:40.362+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:39:40.362] Initiating stopEphemeralMongo [2021-10-22T02:39:40.362+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:39:40.362] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44938 (local=false) [2021-10-22T02:39:40.362+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:39:40.362] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44938 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:39:40.785+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:39:40.785] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.200585ms. [2021-10-22T02:39:40.785+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:39:40.785] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:44938 (local=false) [2021-10-22T02:39:40.785+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:39:40.785] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 20.416µs. [2021-10-22T02:39:40.785+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:39:40.785] Successfully finished running stopEphemeralMongo [2021-10-22T02:39:40.785+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:39:40.785] Successfully stopped existing ephemeral mongod. [2021-10-22T02:39:40.785+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:39:40.785] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:39:40.785+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:39:40.785] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:39:40.821+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:39:40.821] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:39:40.822+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:39:40.822] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36685 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:39:41.264+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:41.264] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:41.400+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:39:41.400] clusterConfig unchanged [2021-10-22T02:39:41.416+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:39:41.416] memory:rss=57466880 alloc=10118792 totalalloc=12048652400 mallocs=153958817 frees=153883544 heapinuse=18128896 heapobjects=75273 gcpausetotalns=1668038583 gcpauselastns=90988 gcnum=1742 utimens=268737169000 stimens=108772374000 minflt=653917 majflt=405 [2021-10-22T02:39:42.082+0000] [.error] [src/action/start.go:func1:93] [103] [02:39:42.082] Error starting mongod : [02:39:42.082] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36685,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36685 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 47091 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:39:40.874+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:39:40.874+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:39:40.874+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36685,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:39:40.875+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:39:40.877+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:39:40.896+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:39:40.919+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:39:40.919+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:39:40.920+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:39:40.968+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:39:40.968+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:39:41.491+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870381:491714][47091:0x7f7018d72bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:39:41.660+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870381:660183][47091:0x7f7018d72bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:39:41.780+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870381:780130][47091:0x7f7018d72bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:39:41.900+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870381:900911][47091:0x7f7018d72bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:39:41.971+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870381:971065][47091:0x7f7018d72bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:39:42.031+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870382:31270][47091:0x7f7018d72bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:39:42.031+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870382:31345][47091:0x7f7018d72bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:39:42.033+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"in...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:39:42.082+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:39:42.082] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36685 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:39:42.082] Error starting mongod : [02:39:42.082] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36685,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36685 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 47091 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:39:40.874+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:39:40.874+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:39:40.874+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36685,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:39:40.875+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:39:40.877+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:39:40.896+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:39:40.919+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:39:40.919+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:39:40.920+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:39:40.968+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:39:40.968+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:39:41.491+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870381:491714][47091:0x7f7018d72bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:39:41.660+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870381:660183][47091:0x7f7018d72bc0], txn-recover: ...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:39:42.082+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:39:42.082] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:39:42.082] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36685 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:39:42.082] Error starting mongod : [02:39:42.082] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36685,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36685 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 47091 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:39:40.874+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:39:40.874+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:39:40.874+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36685,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:39:40.875+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:39:40.877+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:39:40.896+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:39:40.919+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:39:40.919+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:39:40.920+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:39:40.968+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:39:40.968+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:39:41.491+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870381:491714][47091:0x7f7018d72bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:39:41.660+00:00"},"s":"I", "c":"STORAGE", "id":22430, "... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:39:42.083+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:39:42.083] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:39:42.083+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:39:42.083] Initiating stopEphemeralMongo [2021-10-22T02:39:42.083+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:39:42.083] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:39:42.083+0000] [.error] [src/director/director.go:executePlan:962] [02:39:42.083] Failed to apply action. Result = : [02:39:42.082] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:39:42.082] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36685 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:39:42.082] Error starting mongod : [02:39:42.082] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36685,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36685 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 47091 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:39:40.874+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:39:40.874+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:39:40.874+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36685,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:39:40.875+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:39:40.877+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:39:40.896+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:39:40.919+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:39:40.919+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:39:40.920+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:39:40.968+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:39:40.968+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:39:41.491+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870381:491714][47091:0x7f7018d72bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:39:42.083+0000] [.error] [src/director/director.go:planAndExecute:567] [02:39:42.083] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:39:42.083] Failed to apply action. Result = : [02:39:42.082] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:39:42.082] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36685 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:39:42.082] Error starting mongod : [02:39:42.082] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36685,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36685 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 47091 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:39:40.874+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:39:40.874+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:39:40.874+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36685,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:39:40.875+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:39:40.877+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:39:40.896+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:39:40.919+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:39:40.919+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:39:40.920+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:39:40.968+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:39:40.968+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:39:41.491+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger messa... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:39:42.083+0000] [.error] [src/director/director.go:mainLoop:399] [02:39:42.083] Failed to planAndExecute : [02:39:42.083] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:39:42.083] Failed to apply action. Result = : [02:39:42.082] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:39:42.082] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36685 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:39:42.082] Error starting mongod : [02:39:42.082] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36685,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36685 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 47091 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:39:40.874+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:39:40.874+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:39:40.874+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36685,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:39:40.875+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:39:40.877+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:39:40.896+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:39:40.919+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:39:40.919+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:39:40.920+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:39:40.968+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:39:40.968+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:39:41.491+00:00"},"s":"I", "c... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:39:42.265+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:42.265] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:42.629+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:39:42.629] clusterConfig unchanged [2021-10-22T02:39:43.266+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:43.266] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:44.267+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:44.267] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:45.266+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:45.266] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:46.266+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:46.266] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:47.267+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:47.267] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:48.268+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:48.268] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:49.268+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:49.268] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:50.269+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:50.269] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:51.233+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:39:51.233] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.891898712s [2021-10-22T02:39:51.267+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:51.267] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:52.267+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:52.267] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:52.761+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:39:52.761] clusterConfig unchanged [2021-10-22T02:39:53.268+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:53.268] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:54.268+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:54.268] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:55.276+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:55.276] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:56.270+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:56.270] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:57.271+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:57.271] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:58.272+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:58.272] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:39:59.272+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:39:59.272] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:00.273+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:00.273] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:01.281+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:01.281] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:02.273+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:02.273] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:02.883+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:40:02.883] clusterConfig unchanged [2021-10-22T02:40:03.274+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:03.274] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:04.277+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:04.277] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:05.275+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:05.275] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:06.276+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:06.276] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:07.276+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:07.276] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:08.277+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:08.277] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:09.277+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:09.277] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:10.278+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:10.278] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:11.279+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:11.279] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:12.280+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:12.280] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:13.014+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:40:13.014] clusterConfig unchanged [2021-10-22T02:40:13.281+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:13.281] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:14.281+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:14.281] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:15.282+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:15.282] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:16.282+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:16.282] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:17.283+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:17.283] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:18.291+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:18.291] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:19.284+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:19.284] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:20.284+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:20.284] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:21.285+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:21.284] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:22.285+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:22.285] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:23.106+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:40:23.106] clusterConfig unchanged [2021-10-22T02:40:23.285+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:23.285] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:24.286+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:24.286] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:25.287+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:25.287] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:26.287+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:26.287] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:27.288+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:27.288] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:28.288+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:28.288] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:29.296+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:29.296] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:30.290+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:30.289] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:31.289+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:31.289] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:32.290+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:32.290] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:33.219+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:40:33.219] clusterConfig unchanged [2021-10-22T02:40:33.292+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:33.292] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:34.291+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:34.291] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:35.292+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:35.291] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:36.292+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:36.292] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:37.293+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:37.293] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:38.293+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:38.293] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:39.293+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:39.293] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:40.298+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:40.298] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:41.294+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:41.294] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:41.416+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:40:41.416] memory:rss=58032128 alloc=9300264 totalalloc=12094115128 mallocs=154501816 frees=154432902 heapinuse=17195008 heapobjects=68914 gcpausetotalns=1668477464 gcpauselastns=105282 gcnum=1748 utimens=270454223000 stimens=109723663000 minflt=658601 majflt=405 [2021-10-22T02:40:42.255+0000] [.info] [src/director/director.go:computePlan:280] [02:40:42.255] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:40:42.255+0000] [.info] [src/director/director.go:tracef:794] [02:40:42.255] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:40:42.255+0000] [.info] [src/director/director.go:tracef:794] [02:40:42.255] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:40:42.255+0000] [.info] [src/director/director.go:tracef:794] [02:40:42.255] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:40:42.255+0000] [.info] [src/director/director.go:tracef:794] [02:40:42.255] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:40:42.255+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:40:42.255] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:40:42.306+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:42.306] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:42.423+0000] [.info] [src/director/director.go:tracef:794] [02:40:42.423] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:40:42.423+0000] [.info] [src/director/director.go:tracef:794] [02:40:42.423] because [2021-10-22T02:40:42.423+0000] [.info] [src/director/director.go:tracef:794] [02:40:42.423] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:40:42.423+0000] [.info] [src/director/director.go:tracef:794] [02:40:42.423] because ['currentState.CanBounceStop' = true] [2021-10-22T02:40:42.575+0000] [.info] [src/director/director.go:tracef:794] [02:40:42.575] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:40:42.575+0000] [.info] [src/director/director.go:tracef:794] [02:40:42.575] because [2021-10-22T02:40:42.575+0000] [.info] [src/director/director.go:tracef:794] [02:40:42.575] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:40:42.575+0000] [.info] [src/director/director.go:tracef:794] [02:40:42.575] because ['currentState.Up' = false] [2021-10-22T02:40:42.575+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:40:42.575] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:40:42.575+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:40:42.575] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:40:42.652+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:40:42.652] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:40:42.661+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:40:42.661] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:40:42.661+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:40:42.661] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:40:42.662+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:40:42.662] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:40:42.663+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:40:42.663] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:40:42.663+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:40:42.663] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:40:42.663+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:40:42.663] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:40:42.665+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:40:42.665] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:40:42.878+0000] [.info] [src/director/director.go:tracef:794] [02:40:42.878] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:40:42.878+0000] [.info] [src/director/director.go:tracef:794] [02:40:42.878] because ['currentState.Up' = false] [2021-10-22T02:40:43.088+0000] [.info] [src/director/director.go:tracef:794] [02:40:43.088] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:40:43.088+0000] [.info] [src/director/director.go:tracef:794] [02:40:43.088] because ['currentState.Up' = false] [2021-10-22T02:40:43.088+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:40:43.088] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:40:43.297+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:43.297] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:43.317+0000] [.info] [src/director/director.go:tracef:794] [02:40:43.316] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:40:43.317+0000] [.info] [src/director/director.go:tracef:794] [02:40:43.317] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:40:43.401+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:40:43.401] clusterConfig unchanged [2021-10-22T02:40:43.511+0000] [.info] [src/director/director.go:tracef:794] [02:40:43.511] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:40:43.511+0000] [.info] [src/director/director.go:tracef:794] [02:40:43.511] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:40:43.512+0000] [.info] [src/director/director.go:tracef:794] [02:40:43.512] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:40:43.512+0000] [.info] [src/director/director.go:tracef:794] [02:40:43.512] because ['currentState.Up' = false] [2021-10-22T02:40:43.512+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:40:43.512] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:40:43.512+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:40:43.512] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:40:43.543+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:40:43.543] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:40:43.544+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:40:43.544] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42506 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:40:44.301+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:44.301] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:44.313+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:40:44.313] ...Finished start [pid=66331] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42506,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:40:44.484+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:40:44.484] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 170.791326ms. [2021-10-22T02:40:44.484+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:40:44.484] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42506 (local=false) [2021-10-22T02:40:44.484+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:40:44.484] Dropping existing documents from system.replset [2021-10-22T02:40:44.513+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:40:44.513] clusterConfig unchanged [2021-10-22T02:40:44.665+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:40:44.665] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:40:44.678+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:40:44.678] Inserting oplogTruncateAfterPoint document [2021-10-22T02:40:44.678+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:40:44.678] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:40:44.679+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:40:44.679] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:40:44.679+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:40:44.679] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:40:44.695+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:40:44.695] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:40:44.695+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:40:44.695] Setting the minvalid document to [{_id ObjectID("617224ac523370002becc932")} {t -1} {ts {0 1}}] [2021-10-22T02:40:44.708+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:40:44.708] Inserted [{_id ObjectID("617224ac523370002becc932")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:40:44.708+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:40:44.708] Inserting new replica set configuration document. [2021-10-22T02:40:44.708+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:40:44.708] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:40:44.722+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:40:44.722] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:40:44.736+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:40:44.736] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:40:44.736+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:40:44.736] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:40:44.736+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:40:44.736] Initiating stopEphemeralMongo [2021-10-22T02:40:44.736+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:40:44.736] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42506 (local=false) [2021-10-22T02:40:44.736+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:40:44.736] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42506 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:40:45.234+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:40:45.234] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 4.459397ms. [2021-10-22T02:40:45.234+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:40:45.234] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42506 (local=false) [2021-10-22T02:40:45.234+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:40:45.234] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.078µs. [2021-10-22T02:40:45.234+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:40:45.234] Successfully finished running stopEphemeralMongo [2021-10-22T02:40:45.234+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:40:45.234] Successfully stopped existing ephemeral mongod. [2021-10-22T02:40:45.234+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:40:45.234] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:40:45.234+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:40:45.234] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:40:45.266+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:40:45.266] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:40:45.267+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:40:45.267] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37994 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:40:45.299+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:45.299] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:45.648+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:40:45.648] clusterConfig unchanged [2021-10-22T02:40:46.301+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:46.301] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:46.437+0000] [.error] [src/action/start.go:func1:93] [103] [02:40:46.437] Error starting mongod : [02:40:46.437] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37994,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37994 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 67928 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:40:45.319+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:40:45.319+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:40:45.319+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37994,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:40:45.321+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:40:45.322+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:40:45.323+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:40:45.335+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:40:45.335+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:40:45.335+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:40:45.382+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:40:45.382+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:40:45.927+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870445:927977][67928:0x7f2a2cdcbbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:40:46.031+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870446:31834][67928:0x7f2a2cdcbbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:40:46.132+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870446:131981][67928:0x7f2a2cdcbbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T02:40:46.250+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870446:250613][67928:0x7f2a2cdcbbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:40:46.323+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870446:323717][67928:0x7f2a2cdcbbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:40:46.387+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870446:387711][67928:0x7f2a2cdcbbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:40:46.387+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870446:387788][67928:0x7f2a2cdcbbc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:40:46.389+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:40:46.437+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:40:46.437] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37994 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:40:46.437] Error starting mongod : [02:40:46.437] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37994,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37994 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 67928 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:40:45.319+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:40:45.319+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:40:45.319+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37994,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:40:45.321+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:40:45.322+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:40:45.323+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:40:45.335+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:40:45.335+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:40:45.335+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:40:45.382+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:40:45.382+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:40:45.927+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870445:927977][67928:0x7f2a2cdcbbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:40:46.031+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870446:31834][67928:0x7f2a2cdcbbc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:40:46.437+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:40:46.437] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:40:46.437] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37994 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:40:46.437] Error starting mongod : [02:40:46.437] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37994,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37994 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 67928 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:40:45.319+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:40:45.319+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:40:45.319+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37994,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:40:45.321+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:40:45.322+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:40:45.323+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:40:45.335+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:40:45.335+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:40:45.335+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:40:45.382+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:40:45.382+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:40:45.927+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870445:927977][67928:0x7f2a2cdcbbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:40:46.031+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ct... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:40:46.438+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:40:46.438] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:40:46.438+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:40:46.438] Initiating stopEphemeralMongo [2021-10-22T02:40:46.438+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:40:46.438] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:40:46.438+0000] [.error] [src/director/director.go:executePlan:962] [02:40:46.438] Failed to apply action. Result = : [02:40:46.437] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:40:46.437] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37994 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:40:46.437] Error starting mongod : [02:40:46.437] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37994,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37994 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 67928 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:40:45.319+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:40:45.319+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:40:45.319+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37994,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:40:45.321+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:40:45.322+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:40:45.323+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:40:45.335+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:40:45.335+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:40:45.335+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:40:45.382+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:40:45.382+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:40:45.927+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870445:927977][67928:0x7f2a2cdcbbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:40:46.438+0000] [.error] [src/director/director.go:planAndExecute:567] [02:40:46.438] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:40:46.438] Failed to apply action. Result = : [02:40:46.437] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:40:46.437] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37994 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:40:46.437] Error starting mongod : [02:40:46.437] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37994,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37994 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 67928 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:40:45.319+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:40:45.319+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:40:45.319+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37994,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:40:45.321+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:40:45.322+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:40:45.323+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:40:45.335+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:40:45.335+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:40:45.335+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:40:45.382+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:40:45.382+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:40:45.927+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:40:46.438+0000] [.error] [src/director/director.go:mainLoop:399] [02:40:46.438] Failed to planAndExecute : [02:40:46.438] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:40:46.438] Failed to apply action. Result = : [02:40:46.437] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:40:46.437] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:37994 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:40:46.437] Error starting mongod : [02:40:46.437] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37994,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 37994 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 67928 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:40:45.319+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:40:45.319+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:40:45.319+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":37994,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:40:45.321+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:40:45.322+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:40:45.323+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:40:45.335+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:40:45.335+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:40:45.335+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:40:45.382+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:40:45.382+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:40:45.927+00:00"},"s":"I", "c":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:40:46.896+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:40:46.896] clusterConfig unchanged [2021-10-22T02:40:47.299+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:47.299] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:48.300+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:48.300] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:49.300+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:49.300] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:50.302+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:50.302] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:51.302+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:51.302] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:52.309+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:52.309] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:53.310+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:53.310] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:54.310+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:54.310] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:55.310+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:55.310] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:55.474+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:40:55.474] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.808608914s [2021-10-22T02:40:56.307+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:56.307] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:57.057+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:40:57.057] clusterConfig unchanged [2021-10-22T02:40:57.307+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:57.307] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:58.308+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:58.308] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:40:59.308+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:40:59.308] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:00.309+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:00.309] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:01.310+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:01.310] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:02.310+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:02.310] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:03.310+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:03.310] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:04.310+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:04.310] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:05.310+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:05.310] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:06.311+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:06.311] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:07.150+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:41:07.150] clusterConfig unchanged [2021-10-22T02:41:07.312+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:07.312] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:08.313+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:08.313] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:09.313+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:09.313] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:10.313+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:10.313] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:11.314+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:11.314] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:12.318+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:12.318] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:13.316+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:13.316] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:14.317+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:14.317] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:15.317+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:15.317] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:16.318+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:16.318] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:17.272+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:41:17.272] clusterConfig unchanged [2021-10-22T02:41:17.318+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:17.318] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:18.318+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:18.318] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:18.568+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:41:18.568] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:41:18.570+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:18.570] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:41:18.570+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [02:41:18.570] Could not get version : [02:41:18.570] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:41:18.570+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [02:41:18.570] Error getting version for log lines. Skipping : [02:41:18.570] Could not get version : [02:41:18.570] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:41:19.320+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:19.320] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:20.319+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:20.319] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:21.328+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:21.328] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:22.321+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:22.321] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:23.322+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:23.322] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:24.322+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:24.322] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:25.322+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:25.322] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:26.323+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:26.323] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:27.323+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:27.323] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:27.373+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:41:27.373] clusterConfig unchanged [2021-10-22T02:41:28.323+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:28.323] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:29.325+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:29.325] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:30.325+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:30.325] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:31.325+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:31.325] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:32.327+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:32.327] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:33.327+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:33.327] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:34.327+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:34.327] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:35.330+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:35.330] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:36.329+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:36.329] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:37.329+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:37.329] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:37.507+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:41:37.507] clusterConfig unchanged [2021-10-22T02:41:38.329+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:38.329] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:39.329+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:39.329] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:40.330+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:40.330] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:41.330+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:41.330] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:41.417+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:41:41.417] memory:rss=57917440 alloc=14455888 totalalloc=12186880464 mallocs=155706217 frees=155578578 heapinuse=18546688 heapobjects=127639 gcpausetotalns=1670344002 gcpauselastns=47841 gcnum=1759 utimens=273104237000 stimens=110962201000 minflt=664784 majflt=405 [2021-10-22T02:41:42.330+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:42.330] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:43.331+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:43.331] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:44.331+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:44.331] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:44.503+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [02:41:44.503] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:37994 (local=false) because it is not up [2021-10-22T02:41:45.332+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:45.332] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:46.333+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:46.333] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:46.606+0000] [.info] [src/director/director.go:computePlan:280] [02:41:46.606] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:41:46.606+0000] [.info] [src/director/director.go:tracef:794] [02:41:46.606] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:41:46.606+0000] [.info] [src/director/director.go:tracef:794] [02:41:46.606] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:41:46.606+0000] [.info] [src/director/director.go:tracef:794] [02:41:46.606] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:41:46.606+0000] [.info] [src/director/director.go:tracef:794] [02:41:46.606] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:41:46.606+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:41:46.606] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:41:46.752+0000] [.info] [src/director/director.go:tracef:794] [02:41:46.752] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:41:46.752+0000] [.info] [src/director/director.go:tracef:794] [02:41:46.752] because [2021-10-22T02:41:46.752+0000] [.info] [src/director/director.go:tracef:794] [02:41:46.752] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:41:46.752+0000] [.info] [src/director/director.go:tracef:794] [02:41:46.752] because ['currentState.CanBounceStop' = true] [2021-10-22T02:41:46.914+0000] [.info] [src/director/director.go:tracef:794] [02:41:46.914] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:41:46.914+0000] [.info] [src/director/director.go:tracef:794] [02:41:46.914] because [2021-10-22T02:41:46.914+0000] [.info] [src/director/director.go:tracef:794] [02:41:46.914] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:41:46.914+0000] [.info] [src/director/director.go:tracef:794] [02:41:46.914] because ['currentState.Up' = false] [2021-10-22T02:41:46.914+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:41:46.914] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:41:46.914+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:41:46.914] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:41:46.996+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:41:46.996] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:41:46.996+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:41:46.996] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:41:47.004+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:41:47.004] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:41:47.005+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:41:47.005] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:41:47.006+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:41:47.006] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:41:47.006+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:41:47.006] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:41:47.007+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:41:47.007] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:41:47.008+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:41:47.008] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:41:47.222+0000] [.info] [src/director/director.go:tracef:794] [02:41:47.222] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:41:47.222+0000] [.info] [src/director/director.go:tracef:794] [02:41:47.222] because ['currentState.Up' = false] [2021-10-22T02:41:47.337+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:47.337] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:47.450+0000] [.info] [src/director/director.go:tracef:794] [02:41:47.450] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:41:47.450+0000] [.info] [src/director/director.go:tracef:794] [02:41:47.450] because ['currentState.Up' = false] [2021-10-22T02:41:47.450+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:41:47.450] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:41:47.643+0000] [.info] [src/director/director.go:tracef:794] [02:41:47.643] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:41:47.643+0000] [.info] [src/director/director.go:tracef:794] [02:41:47.643] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:41:47.681+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:41:47.681] clusterConfig unchanged [2021-10-22T02:41:47.858+0000] [.info] [src/director/director.go:tracef:794] [02:41:47.858] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:41:47.858+0000] [.info] [src/director/director.go:tracef:794] [02:41:47.858] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:41:47.858+0000] [.info] [src/director/director.go:tracef:794] [02:41:47.858] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:41:47.858+0000] [.info] [src/director/director.go:tracef:794] [02:41:47.858] because ['currentState.Up' = false] [2021-10-22T02:41:47.858+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:41:47.858] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:41:47.858+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:41:47.858] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:41:47.893+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:41:47.893] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:41:47.893+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:41:47.893] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 54170 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:41:48.338+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:48.338] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:48.605+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:41:48.605] ...Finished start [pid=13037] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":54170,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:41:48.735+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:41:48.735] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 130.021295ms. [2021-10-22T02:41:48.735+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:41:48.735] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:54170 (local=false) [2021-10-22T02:41:48.735+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:41:48.735] Dropping existing documents from system.replset [2021-10-22T02:41:48.877+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:41:48.877] clusterConfig unchanged [2021-10-22T02:41:48.878+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:41:48.878] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:41:48.890+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:41:48.889] Inserting oplogTruncateAfterPoint document [2021-10-22T02:41:48.890+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:41:48.890] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:41:48.891+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:41:48.891] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:41:48.891+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:41:48.891] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:41:48.904+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:41:48.904] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:41:48.904+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:41:48.904] Setting the minvalid document to [{_id ObjectID("617224ec523370002becc940")} {t -1} {ts {0 1}}] [2021-10-22T02:41:48.917+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:41:48.917] Inserted [{_id ObjectID("617224ec523370002becc940")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:41:48.917+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:41:48.917] Inserting new replica set configuration document. [2021-10-22T02:41:48.917+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:41:48.917] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:41:48.931+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:41:48.931] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:41:48.935+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:41:48.935] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:41:48.935+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:41:48.935] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:41:48.935+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:41:48.935] Initiating stopEphemeralMongo [2021-10-22T02:41:48.935+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:41:48.935] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:54170 (local=false) [2021-10-22T02:41:48.935+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:41:48.935] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:54170 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:41:49.298+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:41:49.298] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.991365ms. [2021-10-22T02:41:49.298+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:41:49.298] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:54170 (local=false) [2021-10-22T02:41:49.298+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:41:49.298] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.547µs. [2021-10-22T02:41:49.298+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:41:49.298] Successfully finished running stopEphemeralMongo [2021-10-22T02:41:49.298+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:41:49.298] Successfully stopped existing ephemeral mongod. [2021-10-22T02:41:49.298+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:41:49.298] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:41:49.298+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:41:49.298] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:41:49.336+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:41:49.335] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:41:49.336+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:41:49.336] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35445 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:41:49.340+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:49.340] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:50.117+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:41:50.117] clusterConfig unchanged [2021-10-22T02:41:50.340+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:50.340] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:50.571+0000] [.error] [src/action/start.go:func1:93] [103] [02:41:50.571] Error starting mongod : [02:41:50.570] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35445,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35445 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:41:49.369Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 14531 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:41:49.400+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:41:49.400+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:41:49.400+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35445,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:41:49.402+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:41:49.404+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:41:49.405+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:41:49.445+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:41:49.445+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:41:49.458+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:41:49.514+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:41:49.514+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:41:50.022+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870510:22470][14531:0x7fb0a0910bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:41:50.131+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870510:131430][14531:0x7fb0a0910bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:41:50.236+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870510:236867][14531:0x7fb0a0910bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:41:50.355+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870510:355519][14531:0x7fb0a0910bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:41:50.439+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870510:439810][14531:0x7fb0a0910bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:41:50.521+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870510:521788][14531:0x7fb0a0910bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:41:50.521+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870510:521895][14531:0x7fb0a0910bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:41:50.571+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:41:50.571] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35445 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:41:50.571] Error starting mongod : [02:41:50.570] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35445,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35445 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:41:49.369Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 14531 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:41:49.400+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:41:49.400+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:41:49.400+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35445,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:41:49.402+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:41:49.404+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:41:49.405+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:41:49.445+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:41:49.445+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:41:49.458+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:41:49.514+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:41:49.514+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:41:50.022+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870510:22470][14531:0x7fb0a0910bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:41:50.131+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:41:50.571+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:41:50.571] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:41:50.571] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35445 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:41:50.571] Error starting mongod : [02:41:50.570] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35445,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35445 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:41:49.369Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 14531 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:41:49.400+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:41:49.400+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:41:49.400+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35445,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:41:49.402+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:41:49.404+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:41:49.405+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:41:49.445+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:41:49.445+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:41:49.458+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:41:49.514+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:41:49.514+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:41:50.022+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870510:22470][14531:0x7fb0a0910bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:41:50.571+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:41:50.571] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:41:50.571+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:41:50.571] Initiating stopEphemeralMongo [2021-10-22T02:41:50.571+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:41:50.571] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:41:50.575+0000] [.error] [src/director/director.go:executePlan:962] [02:41:50.574] Failed to apply action. Result = : [02:41:50.571] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:41:50.571] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35445 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:41:50.571] Error starting mongod : [02:41:50.570] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35445,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35445 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:41:49.369Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 14531 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:41:49.400+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:41:49.400+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:41:49.400+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35445,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:41:49.402+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:41:49.404+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:41:49.405+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:41:49.445+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:41:49.445+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:41:49.458+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:41:49.514+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:41:49.514+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:41:50.022+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:41:50.576+0000] [.error] [src/director/director.go:planAndExecute:567] [02:41:50.576] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:41:50.574] Failed to apply action. Result = : [02:41:50.571] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:41:50.571] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35445 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:41:50.571] Error starting mongod : [02:41:50.570] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35445,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35445 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:41:49.369Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 14531 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:41:49.400+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:41:49.400+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:41:49.400+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35445,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:41:49.402+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:41:49.404+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:41:49.405+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:41:49.445+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:41:49.445+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:41:49.458+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:41:49.514+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:41:49.514+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-1","msg":"OCSP response","attr":{"status":{"co... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:41:50.576+0000] [.error] [src/director/director.go:mainLoop:399] [02:41:50.576] Failed to planAndExecute : [02:41:50.576] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:41:50.574] Failed to apply action. Result = : [02:41:50.571] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:41:50.571] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:35445 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:41:50.571] Error starting mongod : [02:41:50.570] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35445,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 35445 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:41:49.369Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 14531 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: 1.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:41:49.400+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:41:49.400+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:41:49.400+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":35445,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:41:49.402+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:41:49.404+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:41:49.405+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:41:49.445+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:41:49.445+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:41:49.458+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:41:49.514+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-1","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:41:49.514+00:00"},"s":"I", "c":"NETWORK", "id":57716... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:41:51.339+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:51.339] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:51.377+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:41:51.377] clusterConfig unchanged [2021-10-22T02:41:52.340+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:52.340] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:53.339+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:53.338] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:54.341+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:54.341] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:55.341+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:55.341] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:56.343+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:56.343] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:57.341+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:57.341] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:58.343+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:58.343] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:59.342+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:41:59.342] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:41:59.805+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:41:59.805] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.797010243s [2021-10-22T02:42:00.349+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:00.349] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:01.354+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:01.354] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:01.480+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:42:01.479] clusterConfig unchanged [2021-10-22T02:42:02.341+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:02.341] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:03.342+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:03.342] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:04.343+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:04.343] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:05.343+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:05.343] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:06.344+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:06.344] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:07.345+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:07.345] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:08.352+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:08.352] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:09.350+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:09.350] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:10.351+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:10.351] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:11.359+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:11.358] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:11.583+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:42:11.583] clusterConfig unchanged [2021-10-22T02:42:12.352+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:12.352] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:13.352+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:13.352] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:14.354+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:14.354] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:15.354+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:15.354] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:16.355+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:16.355] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:17.356+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:17.356] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:18.358+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:18.358] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:19.356+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:19.356] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:20.360+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:20.360] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:21.357+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:21.357] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:21.695+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:42:21.695] clusterConfig unchanged [2021-10-22T02:42:22.363+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:22.363] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:23.358+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:23.358] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:24.359+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:24.359] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:25.361+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:25.361] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:26.360+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:26.360] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:27.361+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:27.361] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:27.574+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:42:27.573] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:42:27.574+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:42:27.574] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:42:28.362+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:28.362] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:29.362+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:29.362] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:30.363+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:30.363] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:31.363+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:31.363] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:31.787+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:42:31.787] clusterConfig unchanged [2021-10-22T02:42:32.364+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:32.364] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:33.365+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:33.365] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:34.373+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:34.373] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:35.365+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:35.365] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:36.365+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:36.365] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:37.366+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:37.366] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:38.366+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:38.366] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:39.369+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [02:42:39.369] lsblk cache invalidated, running lsblk command [2021-10-22T02:42:39.370+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:39.370] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:40.368+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:40.368] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:41.368+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:41.368] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:41.418+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:42:41.418] memory:rss=57753600 alloc=13403448 totalalloc=12282573016 mallocs=156935526 frees=156820428 heapinuse=18571264 heapobjects=115098 gcpausetotalns=1672066802 gcpauselastns=217732 gcnum=1771 utimens=275811093000 stimens=112119000000 minflt=669917 majflt=405 [2021-10-22T02:42:41.898+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:42:41.898] clusterConfig unchanged [2021-10-22T02:42:42.369+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:42.369] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:43.376+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:43.376] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:44.370+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:44.370] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:45.371+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:45.371] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:46.371+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:46.371] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:47.372+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:47.372] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:48.373+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:48.373] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:49.373+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:49.373] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:50.374+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:50.374] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:50.758+0000] [.info] [src/director/director.go:computePlan:280] [02:42:50.758] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:42:50.758+0000] [.info] [src/director/director.go:tracef:794] [02:42:50.758] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:42:50.758+0000] [.info] [src/director/director.go:tracef:794] [02:42:50.758] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:42:50.758+0000] [.info] [src/director/director.go:tracef:794] [02:42:50.758] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:42:50.758+0000] [.info] [src/director/director.go:tracef:794] [02:42:50.758] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:42:50.758+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:42:50.758] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:42:50.920+0000] [.info] [src/director/director.go:tracef:794] [02:42:50.920] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:42:50.920+0000] [.info] [src/director/director.go:tracef:794] [02:42:50.920] because [2021-10-22T02:42:50.920+0000] [.info] [src/director/director.go:tracef:794] [02:42:50.920] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:42:50.920+0000] [.info] [src/director/director.go:tracef:794] [02:42:50.920] because ['currentState.CanBounceStop' = true] [2021-10-22T02:42:51.073+0000] [.info] [src/director/director.go:tracef:794] [02:42:51.073] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:42:51.073+0000] [.info] [src/director/director.go:tracef:794] [02:42:51.073] because [2021-10-22T02:42:51.073+0000] [.info] [src/director/director.go:tracef:794] [02:42:51.073] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:42:51.073+0000] [.info] [src/director/director.go:tracef:794] [02:42:51.073] because ['currentState.Up' = false] [2021-10-22T02:42:51.073+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:42:51.073] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:42:51.073+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:42:51.073] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:42:51.154+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:42:51.153] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:42:51.162+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:42:51.162] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:42:51.162+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:42:51.162] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:42:51.163+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:42:51.163] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:42:51.164+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:42:51.164] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:42:51.164+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:42:51.164] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:42:51.164+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:42:51.164] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:42:51.165+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:42:51.165] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:42:51.377+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:51.376] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:51.416+0000] [.info] [src/director/director.go:tracef:794] [02:42:51.416] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:42:51.416+0000] [.info] [src/director/director.go:tracef:794] [02:42:51.416] because ['currentState.Up' = false] [2021-10-22T02:42:51.667+0000] [.info] [src/director/director.go:tracef:794] [02:42:51.667] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:42:51.667+0000] [.info] [src/director/director.go:tracef:794] [02:42:51.667] because ['currentState.Up' = false] [2021-10-22T02:42:51.667+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:42:51.667] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:42:51.880+0000] [.info] [src/director/director.go:tracef:794] [02:42:51.880] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:42:51.881+0000] [.info] [src/director/director.go:tracef:794] [02:42:51.881] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:42:52.089+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:42:52.089] clusterConfig unchanged [2021-10-22T02:42:52.109+0000] [.info] [src/director/director.go:tracef:794] [02:42:52.109] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:42:52.109+0000] [.info] [src/director/director.go:tracef:794] [02:42:52.109] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:42:52.109+0000] [.info] [src/director/director.go:tracef:794] [02:42:52.109] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:42:52.109+0000] [.info] [src/director/director.go:tracef:794] [02:42:52.109] because ['currentState.Up' = false] [2021-10-22T02:42:52.109+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:42:52.109] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:42:52.109+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:42:52.109] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:42:52.142+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:42:52.142] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:42:52.142+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:42:52.142] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 39986 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:42:52.377+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:52.377] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:52.881+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:42:52.881] ...Finished start [pid=33912] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":39986,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:42:53.024+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:42:53.024] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 143.177398ms. [2021-10-22T02:42:53.024+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:42:53.024] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39986 (local=false) [2021-10-22T02:42:53.024+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:42:53.024] Dropping existing documents from system.replset [2021-10-22T02:42:53.166+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:42:53.166] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:42:53.170+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:42:53.170] Inserting oplogTruncateAfterPoint document [2021-10-22T02:42:53.170+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:42:53.170] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:42:53.171+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:42:53.171] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:42:53.171+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:42:53.171] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:42:53.184+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:42:53.184] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:42:53.184+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:42:53.184] Setting the minvalid document to [{_id ObjectID("6172252d523370002becc94e")} {t -1} {ts {0 1}}] [2021-10-22T02:42:53.191+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:42:53.191] clusterConfig unchanged [2021-10-22T02:42:53.197+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:42:53.197] Inserted [{_id ObjectID("6172252d523370002becc94e")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:42:53.197+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:42:53.197] Inserting new replica set configuration document. [2021-10-22T02:42:53.197+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:42:53.197] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:42:53.231+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:42:53.230] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:42:53.234+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:42:53.234] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:42:53.235+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:42:53.235] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:42:53.235+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:42:53.235] Initiating stopEphemeralMongo [2021-10-22T02:42:53.235+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:42:53.235] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39986 (local=false) [2021-10-22T02:42:53.235+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:42:53.235] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39986 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:42:53.376+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:53.376] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:53.668+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:42:53.668] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.356304ms. [2021-10-22T02:42:53.668+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:42:53.668] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:39986 (local=false) [2021-10-22T02:42:53.668+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:42:53.668] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 35.799µs. [2021-10-22T02:42:53.668+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:42:53.668] Successfully finished running stopEphemeralMongo [2021-10-22T02:42:53.668+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:42:53.668] Successfully stopped existing ephemeral mongod. [2021-10-22T02:42:53.668+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:42:53.668] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:42:53.668+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:42:53.668] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:42:53.703+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:42:53.703] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:42:53.703+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:42:53.703] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48857 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:42:54.294+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:42:54.294] clusterConfig unchanged [2021-10-22T02:42:54.377+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:54.377] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:54.889+0000] [.error] [src/action/start.go:func1:93] [103] [02:42:54.889] Error starting mongod : [02:42:54.889] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48857,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48857 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 35345 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:42:53.755+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:42:53.755+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:42:53.755+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48857,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:42:53.756+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:42:53.758+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:42:53.758+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:42:53.831+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:42:53.831+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:42:53.837+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:42:53.890+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:42:53.890+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:42:54.386+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870574:386361][35345:0x7fa063d31bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:42:54.490+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870574:490908][35345:0x7fa063d31bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:42:54.590+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870574:590865][35345:0x7fa063d31bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:42:54.706+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870574:706606][35345:0x7fa063d31bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:42:54.781+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870574:781625][35345:0x7fa063d31bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:42:54.839+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870574:839709][35345:0x7fa063d31bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:42:54.839+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870574:839784][35345:0x7fa063d31bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:42:54.841+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:42:54.889+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:42:54.889] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48857 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:42:54.889] Error starting mongod : [02:42:54.889] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48857,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48857 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 35345 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:42:53.755+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:42:53.755+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:42:53.755+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48857,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:42:53.756+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:42:53.758+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:42:53.758+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:42:53.831+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:42:53.831+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:42:53.837+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:42:53.890+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:42:53.890+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:42:54.386+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870574:386361][35345:0x7fa063d31bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:42:54.490+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870574:490908][35345:0x7fa063d31bc0], txn-recover: [WT...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:42:54.889+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:42:54.889] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:42:54.889] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48857 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:42:54.889] Error starting mongod : [02:42:54.889] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48857,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48857 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 35345 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:42:53.755+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:42:53.755+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:42:53.755+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48857,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:42:53.756+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:42:53.758+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:42:53.758+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:42:53.831+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:42:53.831+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:42:53.837+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:42:53.890+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:42:53.890+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:42:54.386+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870574:386361][35345:0x7fa063d31bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:42:54.490+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:42:54.889+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:42:54.889] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:42:54.889+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:42:54.889] Initiating stopEphemeralMongo [2021-10-22T02:42:54.889+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:42:54.889] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:42:54.890+0000] [.error] [src/director/director.go:executePlan:962] [02:42:54.890] Failed to apply action. Result = : [02:42:54.889] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:42:54.889] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48857 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:42:54.889] Error starting mongod : [02:42:54.889] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48857,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48857 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 35345 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:42:53.755+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:42:53.755+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:42:53.755+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48857,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:42:53.756+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:42:53.758+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:42:53.758+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:42:53.831+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:42:53.831+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:42:53.837+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:42:53.890+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:42:53.890+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:42:54.386+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870574:386361][35345:0x7fa063d31bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:42:54.890+0000] [.error] [src/director/director.go:planAndExecute:567] [02:42:54.890] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:42:54.890] Failed to apply action. Result = : [02:42:54.889] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:42:54.889] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48857 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:42:54.889] Error starting mongod : [02:42:54.889] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48857,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48857 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 35345 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:42:53.755+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:42:53.755+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:42:53.755+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48857,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:42:53.756+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:42:53.758+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:42:53.758+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:42:53.831+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:42:53.831+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:42:53.837+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:42:53.890+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:42:53.890+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:42:54.386+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:42:54.890+0000] [.error] [src/director/director.go:mainLoop:399] [02:42:54.890] Failed to planAndExecute : [02:42:54.890] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:42:54.890] Failed to apply action. Result = : [02:42:54.889] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:42:54.889] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:48857 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:42:54.889] Error starting mongod : [02:42:54.889] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48857,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 48857 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 35345 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:42:53.755+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:42:53.755+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:42:53.755+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":48857,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:42:53.756+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:42:53.758+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:42:53.758+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:42:53.831+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:42:53.831+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:42:53.837+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:42:53.890+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:42:53.890+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:42:54.386+00:00"},"s":"I", "c":"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:42:55.379+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:55.379] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:55.550+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:42:55.550] clusterConfig unchanged [2021-10-22T02:42:56.380+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:56.380] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:57.387+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:57.387] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:58.388+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:58.388] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:42:59.393+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:42:59.393] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:00.391+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:00.391] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:01.395+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:01.395] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:02.395+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:02.395] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:03.394+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:03.394] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:04.290+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:43:04.290] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 13.124287242s [2021-10-22T02:43:04.393+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:04.393] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:05.401+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:05.401] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:05.677+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:43:05.676] clusterConfig unchanged [2021-10-22T02:43:06.394+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:06.394] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:07.394+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:07.394] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:08.394+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:08.394] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:09.394+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:09.394] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:10.394+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:10.394] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:11.395+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:11.395] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:12.395+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:12.395] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:13.396+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:13.396] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:14.397+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:14.397] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:15.398+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:15.398] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:15.769+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:43:15.769] clusterConfig unchanged [2021-10-22T02:43:16.400+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:16.400] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:17.414+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:17.414] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:18.399+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:18.399] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:19.400+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:19.400] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:20.400+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:20.400] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:21.401+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:21.401] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:22.401+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:22.401] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:23.402+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:23.402] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:24.402+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:24.402] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:25.403+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:25.403] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:25.900+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:43:25.900] clusterConfig unchanged [2021-10-22T02:43:26.404+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:26.404] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:27.412+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:27.412] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:28.405+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:28.405] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:29.405+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:29.405] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:30.405+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:30.405] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:31.406+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:31.406] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:32.412+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:32.412] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:33.407+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:33.407] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:34.409+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:34.409] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:35.409+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:35.409] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:36.012+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:43:36.012] clusterConfig unchanged [2021-10-22T02:43:36.409+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:36.409] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:37.409+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:37.409] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:38.410+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:38.410] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:39.411+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:39.411] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:40.412+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:40.412] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:41.413+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:41.413] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:41.419+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:43:41.419] memory:rss=57126912 alloc=10401584 totalalloc=12376574056 mallocs=158151109 frees=158071195 heapinuse=17629184 heapobjects=79914 gcpausetotalns=1673143859 gcpauselastns=55848 gcnum=1783 utimens=278629092000 stimens=113468805000 minflt=675290 majflt=405 [2021-10-22T02:43:42.413+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:42.413] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:43.415+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:43.414] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:44.414+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:44.414] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:45.416+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:45.416] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:46.125+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:43:46.125] clusterConfig unchanged [2021-10-22T02:43:46.416+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:46.416] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:47.424+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:47.424] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:48.417+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:48.417] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:49.418+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:49.418] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:50.418+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:50.418] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:51.419+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:51.419] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:52.419+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:52.419] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:53.420+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:53.420] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:54.420+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:54.420] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:55.060+0000] [.info] [src/director/director.go:computePlan:280] [02:43:55.060] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:43:55.060+0000] [.info] [src/director/director.go:tracef:794] [02:43:55.060] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:43:55.060+0000] [.info] [src/director/director.go:tracef:794] [02:43:55.060] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:43:55.060+0000] [.info] [src/director/director.go:tracef:794] [02:43:55.060] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:43:55.060+0000] [.info] [src/director/director.go:tracef:794] [02:43:55.060] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:43:55.060+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:43:55.060] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:43:55.224+0000] [.info] [src/director/director.go:tracef:794] [02:43:55.224] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:43:55.224+0000] [.info] [src/director/director.go:tracef:794] [02:43:55.224] because [2021-10-22T02:43:55.224+0000] [.info] [src/director/director.go:tracef:794] [02:43:55.224] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:43:55.224+0000] [.info] [src/director/director.go:tracef:794] [02:43:55.224] because ['currentState.CanBounceStop' = true] [2021-10-22T02:43:55.375+0000] [.info] [src/director/director.go:tracef:794] [02:43:55.375] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:43:55.375+0000] [.info] [src/director/director.go:tracef:794] [02:43:55.375] because [2021-10-22T02:43:55.375+0000] [.info] [src/director/director.go:tracef:794] [02:43:55.375] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:43:55.375+0000] [.info] [src/director/director.go:tracef:794] [02:43:55.375] because ['currentState.Up' = false] [2021-10-22T02:43:55.375+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:43:55.375] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:43:55.375+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:43:55.375] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:43:55.420+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:55.420] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:55.459+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:43:55.459] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:43:55.467+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:43:55.467] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:43:55.467+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:43:55.467] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:43:55.467+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:43:55.467] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:43:55.469+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:43:55.469] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:43:55.469+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:43:55.469] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:43:55.469+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:43:55.469] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:43:55.470+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:43:55.470] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:43:55.708+0000] [.info] [src/director/director.go:tracef:794] [02:43:55.708] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:43:55.708+0000] [.info] [src/director/director.go:tracef:794] [02:43:55.708] because ['currentState.Up' = false] [2021-10-22T02:43:55.930+0000] [.info] [src/director/director.go:tracef:794] [02:43:55.930] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:43:55.930+0000] [.info] [src/director/director.go:tracef:794] [02:43:55.930] because ['currentState.Up' = false] [2021-10-22T02:43:55.930+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:43:55.930] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:43:56.142+0000] [.info] [src/director/director.go:tracef:794] [02:43:56.142] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:43:56.142+0000] [.info] [src/director/director.go:tracef:794] [02:43:56.142] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:43:56.308+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:43:56.308] clusterConfig unchanged [2021-10-22T02:43:56.378+0000] [.info] [src/director/director.go:tracef:794] [02:43:56.378] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:43:56.378+0000] [.info] [src/director/director.go:tracef:794] [02:43:56.378] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:43:56.378+0000] [.info] [src/director/director.go:tracef:794] [02:43:56.378] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:43:56.378+0000] [.info] [src/director/director.go:tracef:794] [02:43:56.378] because ['currentState.Up' = false] [2021-10-22T02:43:56.378+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:43:56.378] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:43:56.378+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:43:56.378] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:43:56.413+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:43:56.413] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:43:56.414+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:43:56.414] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 50156 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:43:56.425+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:56.425] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:57.167+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:43:57.167] ...Finished start [pid=54627] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":50156,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:43:57.297+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:43:57.297] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 129.751908ms. [2021-10-22T02:43:57.297+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:43:57.297] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50156 (local=false) [2021-10-22T02:43:57.297+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:43:57.297] Dropping existing documents from system.replset [2021-10-22T02:43:57.424+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:57.424] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:57.427+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:43:57.427] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:43:57.431+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:43:57.431] Inserting oplogTruncateAfterPoint document [2021-10-22T02:43:57.431+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:43:57.431] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:43:57.432+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:43:57.432] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:43:57.432+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:43:57.432] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:43:57.439+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:43:57.439] clusterConfig unchanged [2021-10-22T02:43:57.445+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:43:57.445] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:43:57.445+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:43:57.445] Setting the minvalid document to [{_id ObjectID("6172256d523370002becc95c")} {t -1} {ts {0 1}}] [2021-10-22T02:43:57.458+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:43:57.458] Inserted [{_id ObjectID("6172256d523370002becc95c")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:43:57.458+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:43:57.458] Inserting new replica set configuration document. [2021-10-22T02:43:57.458+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:43:57.458] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:43:57.469+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:43:57.469] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:43:57.473+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:43:57.473] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:43:57.473+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:43:57.473] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:43:57.473+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:43:57.473] Initiating stopEphemeralMongo [2021-10-22T02:43:57.473+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:43:57.473] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50156 (local=false) [2021-10-22T02:43:57.473+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:43:57.473] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50156 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:43:57.843+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:43:57.843] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.248856ms. [2021-10-22T02:43:57.843+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:43:57.843] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:50156 (local=false) [2021-10-22T02:43:57.843+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:43:57.843] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.828µs. [2021-10-22T02:43:57.843+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:43:57.843] Successfully finished running stopEphemeralMongo [2021-10-22T02:43:57.843+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:43:57.843] Successfully stopped existing ephemeral mongod. [2021-10-22T02:43:57.843+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:43:57.843] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:43:57.843+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:43:57.843] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:43:57.874+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:43:57.874] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:43:57.875+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:43:57.875] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57121 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:43:58.426+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:58.426] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:58.560+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:43:58.560] clusterConfig unchanged [2021-10-22T02:43:59.079+0000] [.error] [src/action/start.go:func1:93] [103] [02:43:59.079] Error starting mongod : [02:43:59.078] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57121,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57121 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 56030 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:43:57.931+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:43:57.931+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:43:57.931+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57121,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:43:57.932+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:43:57.933+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:43:57.937+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:43:57.954+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:43:57.954+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:43:57.954+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:43:58.004+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:43:58.004+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:43:58.556+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870638:556379][56030:0x7f6c626c4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:43:58.664+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870638:664020][56030:0x7f6c626c4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:43:58.766+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870638:766955][56030:0x7f6c626c4bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T02:43:58.880+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870638:880199][56030:0x7f6c626c4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:43:58.958+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870638:958350][56030:0x7f6c626c4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:43:59.026+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870639:26519][56030:0x7f6c626c4bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:43:59.026+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870639:26586][56030:0x7f6c626c4bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:43:59.028+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"in...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:43:59.079+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:43:59.079] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:57121 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:43:59.079] Error starting mongod : [02:43:59.078] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57121,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57121 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 56030 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:43:57.931+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:43:57.931+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:43:57.931+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57121,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:43:57.932+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:43:57.933+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:43:57.937+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:43:57.954+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:43:57.954+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:43:57.954+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:43:58.004+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:43:58.004+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:43:58.556+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870638:556379][56030:0x7f6c626c4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:43:58.664+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870638:664020][56030:0x7f6c626c4bc0], txn-recover: ...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:43:59.079+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:43:59.079] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:43:59.079] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:57121 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:43:59.079] Error starting mongod : [02:43:59.078] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57121,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57121 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 56030 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:43:57.931+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:43:57.931+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:43:57.931+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57121,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:43:57.932+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:43:57.933+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:43:57.937+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:43:57.954+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:43:57.954+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:43:57.954+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:43:58.004+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:43:58.004+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:43:58.556+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870638:556379][56030:0x7f6c626c4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:43:58.664+00:00"},"s":"I", "c":"STORAGE", "id":22430, "... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:43:59.079+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:43:59.079] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:43:59.079+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:43:59.079] Initiating stopEphemeralMongo [2021-10-22T02:43:59.079+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:43:59.079] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:43:59.080+0000] [.error] [src/director/director.go:executePlan:962] [02:43:59.080] Failed to apply action. Result = : [02:43:59.079] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:43:59.079] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:57121 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:43:59.079] Error starting mongod : [02:43:59.078] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57121,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57121 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 56030 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:43:57.931+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:43:57.931+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:43:57.931+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57121,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:43:57.932+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:43:57.933+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:43:57.937+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:43:57.954+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:43:57.954+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:43:57.954+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:43:58.004+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:43:58.004+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:43:58.556+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870638:556379][56030:0x7f6c626c4bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:43:59.080+0000] [.error] [src/director/director.go:planAndExecute:567] [02:43:59.080] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:43:59.080] Failed to apply action. Result = : [02:43:59.079] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:43:59.079] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:57121 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:43:59.079] Error starting mongod : [02:43:59.078] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57121,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57121 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 56030 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:43:57.931+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:43:57.931+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:43:57.931+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57121,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:43:57.932+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:43:57.933+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:43:57.937+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:43:57.954+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:43:57.954+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:43:57.954+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:43:58.004+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:43:58.004+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:43:58.556+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger messa... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:43:59.080+0000] [.error] [src/director/director.go:mainLoop:399] [02:43:59.080] Failed to planAndExecute : [02:43:59.080] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:43:59.080] Failed to apply action. Result = : [02:43:59.079] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:43:59.079] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:57121 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:43:59.079] Error starting mongod : [02:43:59.078] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57121,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 57121 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 56030 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:43:57.931+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:43:57.931+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:43:57.931+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":57121,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:43:57.932+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:43:57.933+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:43:57.937+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:43:57.954+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:43:57.954+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:43:57.954+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:43:58.004+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:43:58.004+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:43:58.556+00:00"},"s":"I", "c... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:43:59.427+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:43:59.427] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:43:59.792+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:43:59.792] clusterConfig unchanged [2021-10-22T02:44:00.426+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:00.426] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:01.427+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:01.427] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:02.450+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:02.450] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:03.426+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:03.426] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:04.427+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:04.427] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:05.430+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:05.430] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:06.431+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:06.431] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:07.432+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:07.431] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:08.423+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:44:08.422] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.952152241s [2021-10-22T02:44:08.430+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:08.430] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:09.430+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:09.430] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:09.913+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:44:09.913] clusterConfig unchanged [2021-10-22T02:44:10.430+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:10.430] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:11.431+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:11.431] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:12.438+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:12.438] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:13.432+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:13.432] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:14.432+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:14.432] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:15.432+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:15.432] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:16.432+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:16.432] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:17.434+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:17.434] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:18.434+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:18.434] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:19.435+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:19.435] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:20.114+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:44:20.114] clusterConfig unchanged [2021-10-22T02:44:20.435+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:20.435] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:21.436+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:21.436] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:22.437+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:22.437] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:23.438+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:23.438] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:24.447+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:24.447] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:25.439+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:25.439] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:26.440+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:26.440] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:27.441+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:27.441] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:28.441+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:28.441] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:29.442+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:29.442] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:30.235+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:44:30.235] clusterConfig unchanged [2021-10-22T02:44:30.443+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:30.443] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:31.447+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:31.447] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:32.445+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:32.445] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:33.445+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:33.445] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:34.446+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:34.446] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:35.448+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:35.448] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:36.455+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:36.455] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:37.448+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:37.448] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:38.449+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:38.449] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:39.450+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:39.450] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:40.451+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:40.451] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:40.603+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:44:40.603] clusterConfig unchanged [2021-10-22T02:44:41.420+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:44:41.420] memory:rss=59760640 alloc=11844432 totalalloc=12470461056 mallocs=159365411 frees=159268309 heapinuse=18366464 heapobjects=97102 gcpausetotalns=1674309220 gcpauselastns=109881 gcnum=1794 utimens=281334391000 stimens=114620311000 minflt=680890 majflt=405 [2021-10-22T02:44:41.451+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:41.451] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:42.451+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:42.451] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:43.451+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:43.451] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:44.452+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:44.452] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:44.511+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [02:44:44.511] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:57121 (local=false) because it is not up [2021-10-22T02:44:45.452+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:45.452] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:46.453+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:46.453] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:47.453+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:47.453] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:48.454+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:48.454] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:49.455+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:49.454] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:50.455+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:50.455] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:50.716+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:44:50.716] clusterConfig unchanged [2021-10-22T02:44:51.455+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:51.455] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:52.457+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:52.457] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:53.456+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:53.456] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:54.457+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:54.457] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:55.458+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:55.458] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:56.466+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:56.466] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:57.459+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:57.459] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:58.459+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:58.459] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:59.272+0000] [.info] [src/director/director.go:computePlan:280] [02:44:59.272] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:44:59.272+0000] [.info] [src/director/director.go:tracef:794] [02:44:59.272] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:44:59.272+0000] [.info] [src/director/director.go:tracef:794] [02:44:59.272] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:44:59.272+0000] [.info] [src/director/director.go:tracef:794] [02:44:59.272] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:44:59.272+0000] [.info] [src/director/director.go:tracef:794] [02:44:59.272] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:44:59.272+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:44:59.272] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:44:59.426+0000] [.info] [src/director/director.go:tracef:794] [02:44:59.426] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:44:59.426+0000] [.info] [src/director/director.go:tracef:794] [02:44:59.426] because [2021-10-22T02:44:59.426+0000] [.info] [src/director/director.go:tracef:794] [02:44:59.426] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:44:59.426+0000] [.info] [src/director/director.go:tracef:794] [02:44:59.426] because ['currentState.CanBounceStop' = true] [2021-10-22T02:44:59.468+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:44:59.468] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:44:59.583+0000] [.info] [src/director/director.go:tracef:794] [02:44:59.583] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:44:59.584+0000] [.info] [src/director/director.go:tracef:794] [02:44:59.584] because [2021-10-22T02:44:59.584+0000] [.info] [src/director/director.go:tracef:794] [02:44:59.584] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:44:59.584+0000] [.info] [src/director/director.go:tracef:794] [02:44:59.584] because ['currentState.Up' = false] [2021-10-22T02:44:59.584+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:44:59.584] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:44:59.584+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:44:59.584] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:44:59.667+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:44:59.667] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:44:59.675+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:44:59.675] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:44:59.676+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:44:59.676] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:44:59.677+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:44:59.677] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:44:59.677+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:44:59.677] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:44:59.677+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:44:59.677] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:44:59.679+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:44:59.679] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:44:59.921+0000] [.info] [src/director/director.go:tracef:794] [02:44:59.921] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:44:59.922+0000] [.info] [src/director/director.go:tracef:794] [02:44:59.922] because ['currentState.Up' = false] [2021-10-22T02:45:00.144+0000] [.info] [src/director/director.go:tracef:794] [02:45:00.144] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:45:00.145+0000] [.info] [src/director/director.go:tracef:794] [02:45:00.145] because ['currentState.Up' = false] [2021-10-22T02:45:00.145+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:45:00.145] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:45:00.359+0000] [.info] [src/director/director.go:tracef:794] [02:45:00.359] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:45:00.359+0000] [.info] [src/director/director.go:tracef:794] [02:45:00.359] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:45:00.462+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:00.462] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:00.568+0000] [.info] [src/director/director.go:tracef:794] [02:45:00.568] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:45:00.568+0000] [.info] [src/director/director.go:tracef:794] [02:45:00.568] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:45:00.568+0000] [.info] [src/director/director.go:tracef:794] [02:45:00.568] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:45:00.568+0000] [.info] [src/director/director.go:tracef:794] [02:45:00.568] because ['currentState.Up' = false] [2021-10-22T02:45:00.568+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:45:00.568] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:45:00.568+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:45:00.568] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:45:00.600+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:45:00.600] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:45:00.601+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:45:00.601] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 38137 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:45:00.931+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:45:00.931] clusterConfig unchanged [2021-10-22T02:45:01.375+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:45:01.375] ...Finished start [pid=1291] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":38137,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:45:01.472+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:01.472] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:01.522+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:45:01.522] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 147.124715ms. [2021-10-22T02:45:01.522+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:45:01.522] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38137 (local=false) [2021-10-22T02:45:01.522+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:45:01.522] Dropping existing documents from system.replset [2021-10-22T02:45:01.670+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:45:01.670] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:45:01.686+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:45:01.686] Inserting oplogTruncateAfterPoint document [2021-10-22T02:45:01.686+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:45:01.686] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:45:01.687+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:45:01.687] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:45:01.687+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:45:01.687] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:45:01.700+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:45:01.700] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:45:01.700+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:45:01.700] Setting the minvalid document to [{_id ObjectID("617225ad523370002becc96a")} {t -1} {ts {0 1}}] [2021-10-22T02:45:01.713+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:45:01.713] Inserted [{_id ObjectID("617225ad523370002becc96a")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:45:01.713+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:45:01.713] Inserting new replica set configuration document. [2021-10-22T02:45:01.713+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:45:01.713] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:45:01.739+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:45:01.739] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:45:01.762+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:45:01.762] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:45:01.763+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:45:01.762] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:45:01.763+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:45:01.763] Initiating stopEphemeralMongo [2021-10-22T02:45:01.763+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:45:01.763] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38137 (local=false) [2021-10-22T02:45:01.763+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:45:01.763] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38137 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:45:02.080+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:45:02.080] clusterConfig unchanged [2021-10-22T02:45:02.200+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:45:02.200] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 1.445714ms. [2021-10-22T02:45:02.200+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:45:02.200] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:38137 (local=false) [2021-10-22T02:45:02.200+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:45:02.200] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 34.967µs. [2021-10-22T02:45:02.200+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:45:02.200] Successfully finished running stopEphemeralMongo [2021-10-22T02:45:02.200+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:45:02.200] Successfully stopped existing ephemeral mongod. [2021-10-22T02:45:02.200+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:45:02.200] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:45:02.200+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:45:02.200] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:45:02.237+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:45:02.237] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:45:02.238+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:45:02.238] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36741 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:45:02.473+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:02.473] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:03.232+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:45:03.232] clusterConfig unchanged [2021-10-22T02:45:03.475+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:03.475] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:03.552+0000] [.error] [src/action/start.go:func1:93] [103] [02:45:03.552] Error starting mongod : [02:45:03.552] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36741,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36741 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 2823 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:45:02.303+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:45:02.303+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:45:02.303+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36741,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:45:02.305+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:45:02.307+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:45:02.307+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:45:02.320+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:45:02.320+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:45:02.320+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:45:02.374+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:45:02.374+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:45:02.943+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870702:943875][2823:0x7f2b2403abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:45:03.066+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870703:66335][2823:0x7f2b2403abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:45:03.198+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870703:198928][2823:0x7f2b2403abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:45:03.355+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870703:355662][2823:0x7f2b2403abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:45:03.432+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870703:432476][2823:0x7f2b2403abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:45:03.504+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870703:504321][2823:0x7f2b2403abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:45:03.504+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870703:504397][2823:0x7f2b2403abc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global oldest timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:45:03.505+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"ini...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:45:03.553+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:45:03.553] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36741 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:45:03.552] Error starting mongod : [02:45:03.552] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36741,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36741 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 2823 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:45:02.303+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:45:02.303+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:45:02.303+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36741,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:45:02.305+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:45:02.307+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:45:02.307+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:45:02.320+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:45:02.320+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:45:02.320+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:45:02.374+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:45:02.374+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:45:02.943+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870702:943875][2823:0x7f2b2403abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:45:03.066+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870703:66335][2823:0x7f2b2403abc0], txn-recover...status 14 (warning: log line attempted (13.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:45:03.553+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:45:03.553] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:45:03.553] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36741 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:45:03.552] Error starting mongod : [02:45:03.552] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36741,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36741 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 2823 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:45:02.303+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:45:02.303+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:45:02.303+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36741,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:45:02.305+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:45:02.307+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:45:02.307+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:45:02.320+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:45:02.320+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:45:02.320+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:45:02.374+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:45:02.374+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:45:02.943+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870702:943875][2823:0x7f2b2403abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:45:03.066+00:00"},"s":"I", "c":"STORAGE", "id":22430,... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:45:03.553+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:45:03.553] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:45:03.553+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:45:03.553] Initiating stopEphemeralMongo [2021-10-22T02:45:03.553+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:45:03.553] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:45:03.554+0000] [.error] [src/director/director.go:executePlan:962] [02:45:03.554] Failed to apply action. Result = : [02:45:03.553] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:45:03.553] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36741 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:45:03.552] Error starting mongod : [02:45:03.552] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36741,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36741 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 2823 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:45:02.303+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:45:02.303+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:45:02.303+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36741,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:45:02.305+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:45:02.307+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:45:02.307+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:45:02.320+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:45:02.320+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:45:02.320+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:45:02.374+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:45:02.374+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:45:02.943+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870702:943875][2823:0x7f2b2403abc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:45:03.554+0000] [.error] [src/director/director.go:planAndExecute:567] [02:45:03.554] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:45:03.554] Failed to apply action. Result = : [02:45:03.553] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:45:03.553] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36741 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:45:03.552] Error starting mongod : [02:45:03.552] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36741,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36741 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 2823 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:45:02.303+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:45:02.303+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:45:02.303+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36741,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:45:02.305+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:45:02.307+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:45:02.307+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:45:02.320+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:45:02.320+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:45:02.320+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:45:02.374+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:45:02.374+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:45:02.943+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger ... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:45:03.554+0000] [.error] [src/director/director.go:mainLoop:399] [02:45:03.554] Failed to planAndExecute : [02:45:03.554] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:45:03.554] Failed to apply action. Result = : [02:45:03.553] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:45:03.553] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36741 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:45:03.552] Error starting mongod : [02:45:03.552] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36741,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36741 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): about to fork child process, waiting until server is ready for connections. forked process: 2823 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: fig-00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:45:02.303+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:45:02.303+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:45:02.303+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36741,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:45:02.305+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:45:02.307+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:45:02.307+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:45:02.320+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:45:02.320+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:45:02.320+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:45:02.374+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:45:02.374+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:45:02.943+00:00"},"s":"I"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:45:04.475+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:04.475] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:04.571+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:45:04.571] clusterConfig unchanged [2021-10-22T02:45:05.476+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:05.476] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:06.489+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:06.489] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:07.486+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:07.486] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:08.488+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:08.488] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:09.489+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:09.489] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:10.491+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:10.491] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:11.491+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:11.491] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:12.491+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:12.491] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:12.794+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:45:12.794] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 13.115390861s [2021-10-22T02:45:13.490+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:13.490] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:14.489+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:14.489] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:14.672+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:45:14.672] clusterConfig unchanged [2021-10-22T02:45:15.490+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:15.490] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:16.490+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:16.490] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:17.491+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:17.491] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:18.492+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:18.492] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:19.492+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:19.491] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:20.497+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:20.497] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:21.494+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:21.494] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:22.494+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:22.494] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:23.495+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:23.495] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:24.495+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:24.495] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:24.774+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:45:24.774] clusterConfig unchanged [2021-10-22T02:45:25.495+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:25.495] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:26.496+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:26.495] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:27.497+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:27.497] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:28.497+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:28.497] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:29.498+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:29.498] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:30.498+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:30.498] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:31.502+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:31.502] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:32.499+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:32.499] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:33.500+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:33.500] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:34.500+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:34.500] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:34.916+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:45:34.916] clusterConfig unchanged [2021-10-22T02:45:35.501+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:35.501] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:36.501+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:36.501] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:37.502+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:37.502] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:38.502+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:38.502] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:39.503+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:39.503] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:40.504+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:40.504] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:41.422+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:45:41.422] memory:rss=58765312 alloc=10705192 totalalloc=12565779088 mallocs=160590370 frees=160507296 heapinuse=17793024 heapobjects=83074 gcpausetotalns=1676694906 gcpauselastns=56940 gcnum=1806 utimens=284045673000 stimens=115889355000 minflt=687069 majflt=405 [2021-10-22T02:45:41.505+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:41.505] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:42.505+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:42.505] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:43.505+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:43.505] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:44.505+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:44.505] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:45.018+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:45:45.018] clusterConfig unchanged [2021-10-22T02:45:45.506+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:45.506] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:46.507+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:46.507] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:47.507+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:47.507] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:48.508+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:48.508] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:49.508+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:49.508] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:50.508+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:50.508] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:51.509+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:51.509] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:52.510+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:52.510] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:53.514+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:53.514] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:54.511+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:54.511] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:55.150+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:45:55.150] clusterConfig unchanged [2021-10-22T02:45:55.514+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:55.514] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:56.512+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:56.512] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:57.512+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:57.512] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:58.521+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:58.521] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:45:59.513+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:45:59.513] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:00.513+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:00.513] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:01.514+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:01.514] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:02.514+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:02.514] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:03.515+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:03.515] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:03.798+0000] [.info] [src/director/director.go:computePlan:280] [02:46:03.798] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:46:03.798+0000] [.info] [src/director/director.go:tracef:794] [02:46:03.798] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:46:03.798+0000] [.info] [src/director/director.go:tracef:794] [02:46:03.798] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:46:03.798+0000] [.info] [src/director/director.go:tracef:794] [02:46:03.798] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:46:03.798+0000] [.info] [src/director/director.go:tracef:794] [02:46:03.798] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:46:03.799+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:46:03.798] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:46:03.991+0000] [.info] [src/director/director.go:tracef:794] [02:46:03.991] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:46:03.991+0000] [.info] [src/director/director.go:tracef:794] [02:46:03.991] because [2021-10-22T02:46:03.991+0000] [.info] [src/director/director.go:tracef:794] [02:46:03.991] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:46:03.991+0000] [.info] [src/director/director.go:tracef:794] [02:46:03.991] because ['currentState.CanBounceStop' = true] [2021-10-22T02:46:04.152+0000] [.info] [src/director/director.go:tracef:794] [02:46:04.152] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:46:04.152+0000] [.info] [src/director/director.go:tracef:794] [02:46:04.152] because [2021-10-22T02:46:04.152+0000] [.info] [src/director/director.go:tracef:794] [02:46:04.152] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:46:04.152+0000] [.info] [src/director/director.go:tracef:794] [02:46:04.152] because ['currentState.Up' = false] [2021-10-22T02:46:04.152+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:46:04.152] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:46:04.152+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:46:04.152] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:46:04.238+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:46:04.238] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:46:04.246+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:46:04.246] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:46:04.247+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:46:04.247] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:46:04.248+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:46:04.248] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:46:04.249+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:46:04.249] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:46:04.249+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:46:04.249] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:46:04.249+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:46:04.249] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:46:04.250+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:46:04.250] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:46:04.477+0000] [.info] [src/director/director.go:tracef:794] [02:46:04.477] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:46:04.477+0000] [.info] [src/director/director.go:tracef:794] [02:46:04.477] because ['currentState.Up' = false] [2021-10-22T02:46:04.518+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:04.518] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:04.712+0000] [.info] [src/director/director.go:tracef:794] [02:46:04.712] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:46:04.712+0000] [.info] [src/director/director.go:tracef:794] [02:46:04.712] because ['currentState.Up' = false] [2021-10-22T02:46:04.712+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:46:04.712] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:46:04.913+0000] [.info] [src/director/director.go:tracef:794] [02:46:04.913] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:46:04.913+0000] [.info] [src/director/director.go:tracef:794] [02:46:04.913] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:46:05.141+0000] [.info] [src/director/director.go:tracef:794] [02:46:05.141] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:46:05.142+0000] [.info] [src/director/director.go:tracef:794] [02:46:05.142] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:46:05.142+0000] [.info] [src/director/director.go:tracef:794] [02:46:05.142] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:46:05.142+0000] [.info] [src/director/director.go:tracef:794] [02:46:05.142] because ['currentState.Up' = false] [2021-10-22T02:46:05.142+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:46:05.142] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:46:05.142+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:46:05.142] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:46:05.179+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:46:05.179] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:46:05.180+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:46:05.179] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 43400 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:46:05.351+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:46:05.351] clusterConfig unchanged [2021-10-22T02:46:05.530+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:05.530] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:05.903+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:46:05.903] ...Finished start [pid=22153] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":43400,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:46:06.033+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:46:06.033] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 130.182508ms. [2021-10-22T02:46:06.033+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:46:06.033] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:43400 (local=false) [2021-10-22T02:46:06.033+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:46:06.033] Dropping existing documents from system.replset [2021-10-22T02:46:06.167+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:46:06.167] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:46:06.170+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:46:06.170] Inserting oplogTruncateAfterPoint document [2021-10-22T02:46:06.170+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:46:06.170] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:46:06.171+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:46:06.171] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:46:06.171+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:46:06.171] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:46:06.186+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:46:06.186] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:46:06.186+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:46:06.186] Setting the minvalid document to [{_id ObjectID("617225ee523370002becc978")} {t -1} {ts {0 1}}] [2021-10-22T02:46:06.199+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:46:06.199] Inserted [{_id ObjectID("617225ee523370002becc978")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:46:06.199+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:46:06.199] Inserting new replica set configuration document. [2021-10-22T02:46:06.199+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:46:06.199] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:46:06.213+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:46:06.213] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:46:06.216+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:46:06.216] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:46:06.216+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:46:06.216] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:46:06.216+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:46:06.216] Initiating stopEphemeralMongo [2021-10-22T02:46:06.216+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:46:06.216] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:43400 (local=false) [2021-10-22T02:46:06.217+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:46:06.216] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:43400 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:46:06.482+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:46:06.482] clusterConfig unchanged [2021-10-22T02:46:06.519+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:06.519] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:06.594+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:46:06.594] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.469819ms. [2021-10-22T02:46:06.594+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:46:06.594] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:43400 (local=false) [2021-10-22T02:46:06.594+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:46:06.594] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 38.18µs. [2021-10-22T02:46:06.594+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:46:06.594] Successfully finished running stopEphemeralMongo [2021-10-22T02:46:06.594+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:46:06.594] Successfully stopped existing ephemeral mongod. [2021-10-22T02:46:06.594+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:46:06.594] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:46:06.594+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:46:06.594] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:46:06.632+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:46:06.631] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:46:06.632+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:46:06.632] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36649 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:46:07.523+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:07.523] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:07.605+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:46:07.605] clusterConfig unchanged [2021-10-22T02:46:07.848+0000] [.error] [src/action/start.go:func1:93] [103] [02:46:07.848] Error starting mongod : [02:46:07.848] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36649 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:46:06.666Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 23615 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:46:06.688+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:46:06.688+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:46:06.688+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:46:06.690+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:46:06.691+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:46:06.692+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:46:06.775+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:46:06.775+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:46:06.786+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:46:06.832+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:46:06.832+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:46:07.311+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870767:311100][23615:0x7f53e96c8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:46:07.444+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870767:444045][23615:0x7f53e96c8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:46:07.547+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870767:547522][23615:0x7f53e96c8bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35584 to 2/256"}} {"t":{"$date":"2021-10-22T02:46:07.661+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870767:661847][23615:0x7f53e96c8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:46:07.732+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870767:732029][23615:0x7f53e96c8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:46:07.797+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870767:797374][23615:0x7f53e96c8bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:46:07.797+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870767:797482][23615:0x7f53e96c8bc0], txn-recover: [WT_VERB_RECOVERY | WT_V...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:46:07.848+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:46:07.848] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36649 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:46:07.848] Error starting mongod : [02:46:07.848] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36649 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:46:06.666Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 23615 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:46:06.688+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:46:06.688+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:46:06.688+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:46:06.690+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:46:06.691+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:46:06.692+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:46:06.775+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:46:06.775+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:46:06.786+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:46:06.832+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:46:06.832+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:46:07.311+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870767:311100][23615:0x7f53e96c8bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:46:07.444+00:00"},"s":"I",...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:46:07.848+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:46:07.848] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:46:07.848] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36649 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:46:07.848] Error starting mongod : [02:46:07.848] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36649 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:46:06.666Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 23615 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:46:06.688+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:46:06.688+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:46:06.688+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:46:06.690+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:46:06.691+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:46:06.692+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:46:06.775+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:46:06.775+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:46:06.786+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:46:06.832+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:46:06.832+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:46:07.311+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870767:311100][23615:0x7f53e96c8bc0], txn... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:46:07.849+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:46:07.849] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:46:07.849+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:46:07.849] Initiating stopEphemeralMongo [2021-10-22T02:46:07.849+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:46:07.849] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:46:07.849+0000] [.error] [src/director/director.go:executePlan:962] [02:46:07.849] Failed to apply action. Result = : [02:46:07.848] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:46:07.848] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36649 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:46:07.848] Error starting mongod : [02:46:07.848] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36649 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:46:06.666Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 23615 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:46:06.688+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:46:06.688+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:46:06.688+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:46:06.690+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:46:06.691+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:46:06.692+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:46:06.775+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:46:06.775+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:46:06.786+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:46:06.832+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:46:06.832+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:46:07.311+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:46:07.849+0000] [.error] [src/director/director.go:planAndExecute:567] [02:46:07.849] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:46:07.849] Failed to apply action. Result = : [02:46:07.848] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:46:07.848] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36649 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:46:07.848] Error starting mongod : [02:46:07.848] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36649 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:46:06.666Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 23615 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:46:06.688+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:46:06.688+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:46:06.688+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:46:06.690+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:46:06.691+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:46:06.692+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:46:06.775+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:46:06.775+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:46:06.786+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:46:06.832+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:46:06.832+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"cod... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:46:07.849+0000] [.error] [src/director/director.go:mainLoop:399] [02:46:07.849] Failed to planAndExecute : [02:46:07.849] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:46:07.849] Failed to apply action. Result = : [02:46:07.848] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:46:07.848] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:36649 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:46:07.848] Error starting mongod : [02:46:07.848] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 36649 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:46:06.666Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 23615 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: .qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:46:06.688+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:46:06.688+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:46:06.688+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":36649,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:46:06.690+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:46:06.691+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:46:06.692+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:46:06.775+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:46:06.775+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:46:06.786+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:46:06.832+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:46:06.832+00:00"},"s":"I", "c":"NETWORK", "id":577163... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:46:08.523+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:08.523] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:08.836+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:46:08.836] clusterConfig unchanged [2021-10-22T02:46:09.522+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:09.522] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:10.521+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:10.521] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:11.523+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:11.523] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:12.522+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:12.522] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:13.524+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:13.524] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:14.523+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:14.523] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:15.523+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:15.523] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:16.526+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:16.526] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:17.025+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:46:17.025] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.77513327s [2021-10-22T02:46:17.524+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:17.524] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:18.525+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:18.525] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:18.569+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:46:18.569] AccessLogs Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:46:18.571+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:18.571] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:46:18.571+0000] [.error] [src/mongoctl/processctl.go:Version:1422] [02:46:18.571] Could not get version : [02:46:18.571] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:46:18.571+0000] [.warn] [filebeat/mspooler/accessLogs.go:PreFlushProcessing:147] [02:46:18.571] Error getting version for log lines. Skipping : [02:46:18.571] Could not get version : [02:46:18.571] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=true) is down [2021-10-22T02:46:18.947+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:46:18.947] clusterConfig unchanged [2021-10-22T02:46:19.525+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:19.525] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:20.526+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:20.526] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:21.526+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:21.526] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:22.526+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:22.526] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:23.527+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:23.527] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:24.528+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:24.528] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:25.529+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:25.529] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:26.537+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:26.537] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:27.531+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:27.531] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:28.532+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:28.532] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:29.069+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:46:29.069] clusterConfig unchanged [2021-10-22T02:46:29.533+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:29.533] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:30.534+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:30.534] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:31.533+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:31.533] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:32.534+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:32.534] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:33.534+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:33.534] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:34.535+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:34.535] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:35.535+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:35.535] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:36.542+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:36.542] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:37.536+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:37.536] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:38.546+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:38.546] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:39.191+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:46:39.191] clusterConfig unchanged [2021-10-22T02:46:39.536+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:39.536] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:40.537+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:40.537] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:41.423+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:46:41.423] memory:rss=57352192 alloc=10574072 totalalloc=12661285800 mallocs=161812317 frees=161730208 heapinuse=18014208 heapobjects=82109 gcpausetotalns=1678093938 gcpauselastns=110457 gcnum=1818 utimens=286605113000 stimens=117098639000 minflt=692319 majflt=405 [2021-10-22T02:46:41.538+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:41.538] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:42.538+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:42.538] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:43.543+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:43.543] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:44.544+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:44.544] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:45.540+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:45.540] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:46.541+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:46.541] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:47.550+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:47.550] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:48.542+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:48.542] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:49.332+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:46:49.332] clusterConfig unchanged [2021-10-22T02:46:49.543+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:49.543] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:50.544+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:50.544] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:51.544+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:51.544] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:52.545+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:52.544] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:53.545+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:53.545] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:54.546+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:54.545] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:55.546+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:55.546] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:56.547+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:56.547] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:57.548+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:57.548] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:58.548+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:58.548] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:46:59.424+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:46:59.424] clusterConfig unchanged [2021-10-22T02:46:59.551+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:46:59.551] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:00.550+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:00.550] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:01.550+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:01.550] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:02.553+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:02.553] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:03.551+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:03.551] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:04.552+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:04.552] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:05.553+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:05.553] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:06.553+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:06.553] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:07.553+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:07.553] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:08.016+0000] [.info] [src/director/director.go:computePlan:280] [02:47:08.016] ... process has a plan : RestoreRsMemberFromBackupPhase1,RestoreRsMemberFromBackupPhase2,RestoreConfigServerFromBackupPhase3,WaitFeatureCompatibilityVersionCorrect [2021-10-22T02:47:08.016+0000] [.info] [src/director/director.go:tracef:794] [02:47:08.016] Running step: 'CheckRsMemberNeedsRestorePhase1' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:47:08.016+0000] [.info] [src/director/director.go:tracef:794] [02:47:08.016] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase = 0 is <= phase = 0' = true] ] [2021-10-22T02:47:08.017+0000] [.info] [src/director/director.go:tracef:794] [02:47:08.017] Running step: 'MarkRestoreStarted' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:47:08.017+0000] [.info] [src/director/director.go:tracef:794] [02:47:08.017] because [All the following are true: ['desiredState.ProcessType' = mongod] ['currentState.BackupRestoreCorrect' = false] ] [2021-10-22T02:47:08.017+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:47:08.017] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:47:08.178+0000] [.info] [src/director/director.go:tracef:794] [02:47:08.177] Running step: 'WaitCanBounceStop' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:47:08.178+0000] [.info] [src/director/director.go:tracef:794] [02:47:08.178] because [2021-10-22T02:47:08.178+0000] [.info] [src/director/director.go:tracef:794] [02:47:08.178] Running step: 'BounceStopIfUpWithForceKill' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:47:08.178+0000] [.info] [src/director/director.go:tracef:794] [02:47:08.178] because ['currentState.CanBounceStop' = true] [2021-10-22T02:47:08.342+0000] [.info] [src/director/director.go:tracef:794] [02:47:08.342] Running step: 'WaitReadyToRestoreBackup' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:47:08.342+0000] [.info] [src/director/director.go:tracef:794] [02:47:08.342] because [2021-10-22T02:47:08.342+0000] [.info] [src/director/director.go:tracef:794] [02:47:08.342] Running step: 'MakeBackupDataAvailable' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:47:08.342+0000] [.info] [src/director/director.go:tracef:794] [02:47:08.342] because ['currentState.Up' = false] [2021-10-22T02:47:08.342+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:208] [02:47:08.342] SourceDbPath: TargetDbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:47:08.342+0000] [.info] [src/action/restorefrombackup.go:renameDataDirectory:211] [02:47:08.342] Source dbpath doesn't exist - not renaming directory. Names found: [bumper1.dat bumper2.dat bumper3.dat .bumper tmp atlas-4wnh1p-config-0-node-1] [2021-10-22T02:47:08.424+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:47:08.424] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf [2021-10-22T02:47:08.433+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:47:08.433] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/journal [2021-10-22T02:47:08.433+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:298] [02:47:08.433] Not deleting Allow Listed file: /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation_restore_started [2021-10-22T02:47:08.434+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:47:08.434] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1/diagnostic.data [2021-10-22T02:47:08.435+0000] [.info] [src/action/restorefrombackup.go:filterDirFiles:359] [02:47:08.435] Filtering files complete, path = /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:47:08.435+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:47:08.435] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:47:08.435+0000] [.info] [src/util/io.go:PreWarmDisk:29] [02:47:08.435] Pre-warming disk containing /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:47:08.437+0000] [.info] [src/util/io.go:PreWarmDisk:45] [02:47:08.437] Pre-warming starting for disk /dev/nvme1n1 at path /srv/mongodb/atlas-4wnh1p-config-0-node-1 [2021-10-22T02:47:08.557+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:08.557] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:08.641+0000] [.info] [src/director/director.go:tracef:794] [02:47:08.641] Running step: 'PrepareRsForApplyOps' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:47:08.642+0000] [.info] [src/director/director.go:tracef:794] [02:47:08.642] because ['currentState.Up' = false] [2021-10-22T02:47:08.839+0000] [.info] [src/director/director.go:tracef:794] [02:47:08.839] Running step: 'ApplyOpsToDesiredTime' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:47:08.839+0000] [.info] [src/director/director.go:tracef:794] [02:47:08.839] because ['currentState.Up' = false] [2021-10-22T02:47:08.839+0000] [.info] [src/action/restorefrombackup.go:ApplyOpsToDesiredTime:432] [02:47:08.839] Skipping ApplyOpsToDesiredTime because not client PIT restore [2021-10-22T02:47:09.064+0000] [.info] [src/director/director.go:tracef:794] [02:47:09.064] Running step: 'MarkRestorePhase1Finished' of move 'RestoreRsMemberFromBackupPhase1' [2021-10-22T02:47:09.064+0000] [.info] [src/director/director.go:tracef:794] [02:47:09.064] because ['desiredState.ProcessType' = mongod] [2021-10-22T02:47:09.255+0000] [.info] [src/director/director.go:tracef:794] [02:47:09.255] Running step: 'CheckRsMemberNeedsRestorePhase2' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:47:09.255+0000] [.info] [src/director/director.go:tracef:794] [02:47:09.255] because [All the following are true: [All the following are true: ['currentState.BackupRestoreCorrect' = false] ['desiredState.IsCancelled()' = false] ['desiredState.GetRestoreKey()' != ('desiredState.GetRestoreKey()' = V7x5ufjNL2mhimZS5pU6UAjp)] ] ['desiredState.IsArbiter' = false] ['desiredState.ProcessType' = mongod] ['currentState.BackupRestorePhase' = 'Restore Phase 1 Finished'; both are (1)] ] [2021-10-22T02:47:09.255+0000] [.info] [src/director/director.go:tracef:794] [02:47:09.255] Running step: 'SeedRestore' of move 'RestoreRsMemberFromBackupPhase2' [2021-10-22T02:47:09.255+0000] [.info] [src/director/director.go:tracef:794] [02:47:09.255] because ['currentState.Up' = false] [2021-10-22T02:47:09.255+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1156] [02:47:09.255] Initiating startEphemeralMongoAsStandalone [2021-10-22T02:47:09.255+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandalone:200] [02:47:09.255] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:47:09.287+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:47:09.287] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:47:09.288+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:47:09.288] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42659 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 reportOpWriteConcernCountersInServerStatus: "true" suppressNoTLSPeerCertificateWarning: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:47:09.558+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:09.558] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:09.635+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:47:09.635] clusterConfig unchanged [2021-10-22T02:47:10.031+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:377] [02:47:10.031] ...Finished start [pid=42683] cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] with stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42659,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","reportOpWriteConcernCountersInServerStatus":"true","suppressNoTLSPeerCertificateWarning":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false], bin="/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin", executable=/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod [2021-10-22T02:47:10.171+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:47:10.171] Slept until start.go's sleepUntilProcessUp (function com.tengen/cm/action.sleepUntilProcessUp.func1) returned true. It took 139.788103ms. [2021-10-22T02:47:10.171+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoAsStandalone:1170] [02:47:10.171] Successfully finished running startEphemeralMongoAsStandalone. Ephemeral Connection Params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42659 (local=false) [2021-10-22T02:47:10.171+0000] [.info] [src/action/restorefrombackup.go:dropReplicaSetConfigurationDocuments:1331] [02:47:10.171] Dropping existing documents from system.replset [2021-10-22T02:47:10.325+0000] [.info] [src/action/restorefrombackup.go:dropWiredTigerCheckpointDocuments:1336] [02:47:10.325] Dropping replication-related collections from local - replset.oplogTruncateAfterPoint, replset.minvalid, replset.election, replset.initialSyncId [2021-10-22T02:47:10.329+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1394] [02:47:10.329] Inserting oplogTruncateAfterPoint document [2021-10-22T02:47:10.329+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1396] [02:47:10.329] logicalTruncateAfterPoint is: &{1634858228 1} [2021-10-22T02:47:10.330+0000] [.warn] [src/action/restorefrombackup.go:shouldSkipInsertingOplogTruncateTimestamp:1448] [02:47:10.330] Could not find the latest oplog in shouldSkipInsertingOplogTruncateTimestamp [2021-10-22T02:47:10.330+0000] [.info] [src/action/restorefrombackup.go:insertTruncateAfterPointDoc:1427] [02:47:10.330] Setting the oplogTruncateAfterPoint document to {1634858228 1} [2021-10-22T02:47:10.343+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:47:10.343] Inserted [{_id oplogTruncateAfterPoint} {oplogTruncateAfterPoint {1634858228 1}}] into local.replset.oplogTruncateAfterPoint [2021-10-22T02:47:10.343+0000] [.info] [src/action/restorefrombackup.go:insertMinValidDocument:1476] [02:47:10.343] Setting the minvalid document to [{_id ObjectID("6172262e523370002becc986")} {t -1} {ts {0 1}}] [2021-10-22T02:47:10.355+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:47:10.355] Inserted [{_id ObjectID("6172262e523370002becc986")} {t -1} {ts {0 1}}] into local.replset.minvalid [2021-10-22T02:47:10.355+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1528] [02:47:10.355] Inserting new replica set configuration document. [2021-10-22T02:47:10.355+0000] [.info] [src/action/restorefrombackup.go:insertNewRsConf:1545] [02:47:10.355] Inserting rs conf = map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] [2021-10-22T02:47:10.366+0000] [.info] [src/mongoctl/processctl.go:Insert:3070] [02:47:10.366] Inserted map[_id:atlas-4wnh1p-config-0 configsvr:true members:[map[_id:0 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-00.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-00-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-00.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-00.qzt8l.mmscloudteam.com:27017 priority:6.5 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:1 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-01.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-01-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-01.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1] map[_id:2 arbiterOnly:false buildIndexes:true hidden:false horizons:map[MESH:cluster0-config-00-02.qzt8l.mesh.mmscloudteam.com:30000 PRIVATE:cluster0-config-00-02-pri.qzt8l.mmscloudteam.com:27017 PUBLIC:cluster0-config-00-02.qzt8l.mmscloudteam.com:27017] host:atlas-4wnh1p-config-00-02.qzt8l.mmscloudteam.com:27017 priority:7 secondaryDelaySecs:0 tags:map[nodeType:ELECTABLE provider:AWS region:US_WEST_1 workloadType:OPERATIONAL] votes:1]] protocolVersion:1 settings:map[chainingAllowed:false electionTimeoutMillis:10000] version:3] into local.system.replset [2021-10-22T02:47:10.370+0000] [.info] [src/action/restorefrombackup.go:handleRestartOplogParamsIfApplicable:1573] [02:47:10.370] handleRestartOplogParamsIfApplicable - fcv is 5.0 [2021-10-22T02:47:10.370+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1604] [02:47:10.370] Starting restarting ephemeral mongod with oplog replay parameters. [2021-10-22T02:47:10.370+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:47:10.370] Initiating stopEphemeralMongo [2021-10-22T02:47:10.370+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1219] [02:47:10.370] Stopping ephemeral mongo with connection params :atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42659 (local=false) [2021-10-22T02:47:10.370+0000] [.info] [src/mongoctl/processctl.go:StopMongoWithTimeout:655] [02:47:10.370] Stopping mongo with connParams=atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42659 (local=false) (force=false, killIfTimesOut=false)... [2021-10-22T02:47:10.559+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:10.559] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:10.729+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:47:10.729] Slept until ShutdownMongo (function com.tengen/cm/mongoctl.ShutdownMongo.func1) returned true. It took 2.006009ms. [2021-10-22T02:47:10.730+0000] [.info] [src/mongoctl/processctl.go:ShutdownMongo:1032] [02:47:10.730] Shut down mongo listening on atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42659 (local=false) [2021-10-22T02:47:10.730+0000] [.info] [src/process/process.go:SleepUntilTrue:47] [02:47:10.730] Slept until StopMongo (function com.tengen/cm/mongoctl.StopMongoWithTimeout.func1) returned true. It took 37.933µs. [2021-10-22T02:47:10.730+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1231] [02:47:10.730] Successfully finished running stopEphemeralMongo [2021-10-22T02:47:10.730+0000] [.info] [src/action/restorefrombackup.go:restartWithAndWithoutOplogReplayParameters:1608] [02:47:10.730] Successfully stopped existing ephemeral mongod. [2021-10-22T02:47:10.730+0000] [.info] [src/action/restorefrombackup.go:startEphemeralMongoWithOplogReplayParams:1177] [02:47:10.730] Initiating startEphemeralMongoWithOplogReplayParams [2021-10-22T02:47:10.730+0000] [.info] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:163] [02:47:10.730] Starting mongo as standalone on ephemeral port with auth = false ... [2021-10-22T02:47:10.764+0000] [.info] [src/mongoctl/processctl.go:newStartProcessCmd:560] [02:47:10.764] Running command /var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod with args [-f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] [2021-10-22T02:47:10.764+0000] [.info] [src/mongoctl/processctl.go:StartMongoInBin:171] [02:47:10.764] Starting mongod! cmd=Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf] ConfigFile : # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42155 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log [2021-10-22T02:47:10.775+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:47:10.775] clusterConfig unchanged [2021-10-22T02:47:11.570+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:11.570] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:11.871+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:47:11.871] clusterConfig unchanged [2021-10-22T02:47:12.058+0000] [.error] [src/action/start.go:func1:93] [103] [02:47:12.058] Error starting mongod : [02:47:12.058] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42155,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42155 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:47:10.795Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 44078 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:47:10.813+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:47:10.813+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:47:10.813+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42155,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:47:10.815+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:47:10.817+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:47:10.828+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:47:10.845+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:47:10.845+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:47:10.845+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:47:10.891+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:47:10.891+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:47:11.473+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870831:473845][44078:0x7f92e4e95bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:47:11.596+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870831:596520][44078:0x7f92e4e95bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:47:11.719+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870831:719873][44078:0x7f92e4e95bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Main recovery loop: starting at 1/35328 to 2/256"}} {"t":{"$date":"2021-10-22T02:47:11.864+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870831:864928][44078:0x7f92e4e95bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:47:11.945+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870831:945926][44078:0x7f92e4e95bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 2 through 2"}} {"t":{"$date":"2021-10-22T02:47:12.005+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870832:5222][44078:0x7f92e4e95bc0], txn-recover: [WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS] Set global recovery timestamp: (0, 0)"}} {"t":{"$date":"2021-10-22T02:47:12.005+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870832:5300][44078:0x7f92e4e95bc0], txn-recover: [WT_VERB_RECOVERY | WT...6, "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"} : exit status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) [2021-10-22T02:47:12.058+0000] [.error] [src/action/helpers.go:StartMongoOnEphemeralPort:310] [02:47:12.058] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42155 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:47:12.058] Error starting mongod : [02:47:12.058] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42155,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42155 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:47:10.795Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 44078 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:47:10.813+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:47:10.813+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:47:10.813+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42155,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:47:10.815+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:47:10.817+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:47:10.828+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:47:10.845+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:47:10.845+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:47:10.845+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:47:10.891+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:47:10.891+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:47:11.473+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870831:473845][44078:0x7f92e4e95bc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 1 through 2"}} {"t":{"$date":"2021-10-22T02:47:11.596+00:00"},"s...status 14 (warning: log line attempted (13.9k) over max size (10.0k), printing beginning and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) [2021-10-22T02:47:12.059+0000] [.error] [src/action/helpers.go:startMongoOnEphemeralPortAsStandaloneWithOplogReplayParams:193] [02:47:12.059] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:47:12.058] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42155 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:47:12.058] Error starting mongod : [02:47:12.058] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42155,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42155 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:47:10.795Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 44078 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:47:10.813+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:47:10.813+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:47:10.813+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42155,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:47:10.815+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:47:10.817+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:47:10.828+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:47:10.845+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:47:10.845+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:47:10.845+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:47:10.891+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:47:10.891+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:47:11.473+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1634870831:473845][44078:0x7f92e4e95bc0... and end) (warning: log line attempted (11.7k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:47:12.059+0000] [.info] [src/action/helpers.go:touchMarkerFile:717] [02:47:12.059] Marker file /srv/mongodb/atlas-4wnh1p-config-0-node-1 created [2021-10-22T02:47:12.059+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1213] [02:47:12.059] Initiating stopEphemeralMongo [2021-10-22T02:47:12.059+0000] [.info] [src/action/restorefrombackup.go:stopEphemeralMongo:1215] [02:47:12.059] There is no ephemeral param to connect to. stopEphemeralMongo play is a no-op. [2021-10-22T02:47:12.059+0000] [.error] [src/director/director.go:executePlan:962] [02:47:12.059] Failed to apply action. Result = : [02:47:12.059] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:47:12.058] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42155 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:47:12.058] Error starting mongod : [02:47:12.058] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42155,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42155 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:47:10.795Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 44078 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:47:10.813+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:47:10.813+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:47:10.813+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42155,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:47:10.815+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:47:10.817+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:47:10.828+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:47:10.845+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:47:10.845+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:47:10.845+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:47:10.891+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:47:10.891+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:47:11.473+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"initandlisten"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:47:12.060+0000] [.error] [src/director/director.go:planAndExecute:567] [02:47:12.060] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:47:12.059] Failed to apply action. Result = : [02:47:12.059] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:47:12.058] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42155 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:47:12.058] Error starting mongod : [02:47:12.058] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42155,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42155 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:47:10.795Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 44078 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:47:10.813+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:47:10.813+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:47:10.813+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42155,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:47:10.815+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:47:10.817+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:47:10.828+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:47:10.845+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:47:10.845+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:47:10.845+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:47:10.891+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:47:10.891+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status"... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:47:12.060+0000] [.error] [src/director/director.go:mainLoop:399] [02:47:12.060] Failed to planAndExecute : [02:47:12.060] Plan execution failed on step SeedRestore as part of move RestoreRsMemberFromBackupPhase2 : [02:47:12.059] Failed to apply action. Result = : [02:47:12.059] Error starting mongo process atlas-4wnh1p-config-0-node-1 on ephermal port : [02:47:12.058] Error starting mongo with args = ProcArgs{args=map[AUTH_MECHANISMS:SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 BIND_IP:0.0.0.0 DBPATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1 DISABLE_LOGICAL_SESSION_CACHE_REFRESH:true HONOR_SYSTEM_UMASK:false JAVASCRIPT_ENABLED:true LOG_APPEND:true LOG_DESTINATION:file LOG_PATH:/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log MAX_INCOMING_CONNECTIONS:1500 MAX_INCOMING_CONNECTIONS_OVERRIDE:[192.168.248.0/21] PORT:42155 STARTUP_RECOVER_FOR_RESTORE:true RECOVER_FROM_OPLOG_AS_STANDALONE:true TAKE_UNSTABLE_CHECKPOINT_ON_SHUTDOWN:true TLS_ALLOW_CONNECTIONS_WITHOUT_CERTIFICATES:true TLS_CA_FILE:/etc/pki/tls/certs/atlas-bundle.crt TLS_CERTIFICATE_KEY_FILE:/etc/pki/tls/private/mongod.pem TLS_CLUSTER_CA_FILE:/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt TLS_DISABLED_PROTOCOLS:TLS1_0,TLS1_1 TLS_MODE:requireTLS TLS_WITHHOLD_CLIENT_CERTIFICATE:true TTL_MONITOR_ENABLED:false net.compression.compressors:snappy,zstd,zlib setParameter.reportOpWriteConcernCountersInServerStatus:true setParameter.tlsCATrusts:[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}] setParameter.suppressNoTLSPeerCertificateWarning:true setParameter.allowRolesFromX509Certificates:true setParameter.awsSTSUrl:https://sts.us-west-1.amazonaws.com setParameter.mongotHost:localhost:28000 setParameter.watchdogPeriodSeconds:60 storage.wiredTiger.engineConfig.configString:cache_size=512MB] version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise)} : (res=) : [02:47:12.058] Error starting mongod : [02:47:12.058] Error running start command. cmd=[Args=[/var/lib/mongodb-mms-automation/mongodb-linux-x86_64-5.0.3-ent/bin/mongod -f /srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf]], stip=[args={"net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42155,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}[],confPath=/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf,version=5.0.3-ent-657fea5a61a74d7a79df7aff8e4bcf0bc742b748(enterprise),isKmipRotateMasterKey=false,useOldConfFile=false] , Config Used: # THIS FILE IS MAINTAINED BY http://mongodb-cloud-chuck-zhang.ngrok.io . DO NOT MODIFY AS IT WILL BE OVERWRITTEN. # To make changes to your MongoDB deployment, please visit http://mongodb-cloud-chuck-zhang.ngrok.io . Your Group ID is 6170ffbbca5a3038658c8236 . net: bindIp: 0.0.0.0 compression: compressors: snappy,zstd,zlib maxIncomingConnections: 1500 maxIncomingConnectionsOverride: - 192.168.248.0/21 port: 42155 tls: CAFile: /etc/pki/tls/certs/atlas-bundle.crt allowConnectionsWithoutCertificates: true certificateKeyFile: /etc/pki/tls/private/mongod.pem clusterCAFile: /var/lib/mongodb-mms-automation/atlas-cluster-managed.crt disabledProtocols: TLS1_0,TLS1_1 mode: requireTLS processManagement: fork: "true" security: javascriptEnabled: true setParameter: allowRolesFromX509Certificates: "true" authenticationMechanisms: SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509 awsSTSUrl: https://sts.us-west-1.amazonaws.com disableLogicalSessionCacheRefresh: "true" honorSystemUmask: "false" mongotHost: localhost:28000 recoverFromOplogAsStandalone: "true" reportOpWriteConcernCountersInServerStatus: "true" startupRecoveryForRestore: "true" suppressNoTLSPeerCertificateWarning: "true" takeUnstableCheckpointOnShutdown: "true" tlsCATrusts: '[{sha256: "76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633", roles: [{ role: "", db: ""}]}]' tlsWithholdClientCertificate: "true" ttlMonitorEnabled: "false" watchdogPeriodSeconds: "60" storage: dbPath: /srv/mongodb/atlas-4wnh1p-config-0-node-1 engine: wiredTiger wiredTiger: engineConfig: configString: cache_size=512MB systemLog: destination: file logAppend: true path: /srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log - Output (stdout/stderr): {"t":{"$date":"2021-10-22T02:47:10.795Z"},"s":"I", "c":"STORAGE", "id":22293, "ctx":"-","msg":"Engine custom option","attr":{"option":"cache_size=512MB"}} about to fork child process, waiting until server is ready for connections. forked process: 44078 ERROR: child process failed, exited with 14 To see additional information in this output, start without the "--fork" option. - Mongo Logs: -00-01.qzt8l.mmscloudteam.com"}} {"t":{"$date":"2021-10-22T02:47:10.813+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.3","gitVersion":"657fea5a61a74d7a79df7aff8e4bcf0bc742b748","openSSLVersion":"OpenSSL 1.0.1e-fips 11 Feb 2013","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"rhel70","distarch":"x86_64","target_arch":"x86_64"}}}} {"t":{"$date":"2021-10-22T02:47:10.813+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"CentOS Linux release 7.9.2009 (Core)","version":"Kernel 3.10.0-1160.41.1.el7.x86_64"}}} {"t":{"$date":"2021-10-22T02:47:10.813+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/automation-mongod.conf","net":{"bindIp":"0.0.0.0","compression":{"compressors":"snappy,zstd,zlib"},"maxIncomingConnections":1500,"maxIncomingConnectionsOverride":["192.168.248.0/21"],"port":42155,"tls":{"CAFile":"/etc/pki/tls/certs/atlas-bundle.crt","allowConnectionsWithoutCertificates":true,"certificateKeyFile":"/etc/pki/tls/private/mongod.pem","clusterCAFile":"/var/lib/mongodb-mms-automation/atlas-cluster-managed.crt","disabledProtocols":"TLS1_0,TLS1_1","mode":"requireTLS"}},"processManagement":{"fork":true},"security":{"javascriptEnabled":true},"setParameter":{"allowRolesFromX509Certificates":"true","authenticationMechanisms":"SCRAM-SHA-1,MONGODB-AWS,MONGODB-X509","awsSTSUrl":"https://sts.us-west-1.amazonaws.com","disableLogicalSessionCacheRefresh":"true","honorSystemUmask":"false","mongotHost":"localhost:28000","recoverFromOplogAsStandalone":"true","reportOpWriteConcernCountersInServerStatus":"true","startupRecoveryForRestore":"true","suppressNoTLSPeerCertificateWarning":"true","takeUnstableCheckpointOnShutdown":"true","tlsCATrusts":"[{sha256: \"76D7D7A042F4AAC3BFCF548DD0BD9912B1370854AEEFC4C69D88532654F11633\", roles: [{ role: \"\", db: \"\"}]}]","tlsWithholdClientCertificate":"true","ttlMonitorEnabled":"false","watchdogPeriodSeconds":"60"},"storage":{"dbPath":"/srv/mongodb/atlas-4wnh1p-config-0-node-1","engine":"wiredTiger","wiredTiger":{"engineConfig":{"configString":"cache_size=512MB"}}},"systemLog":{"destination":"file","logAppend":true,"path":"/srv/mongodb/atlas-4wnh1p-config-0-node-1/mongodb.log"}}}} {"t":{"$date":"2021-10-22T02:47:10.815+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"initandlisten","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:47:10.817+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=387M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],cache_size=512MB"}} {"t":{"$date":"2021-10-22T02:47:10.828+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"CurlConnPool-0","msg":"Connecting","attr":{"hostAndPort":"r3.o.lencr.org:80"}} {"t":{"$date":"2021-10-22T02:47:10.845+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:47:10.845+00:00"},"s":"I", "c":"NETWORK", "id":577163, "ctx":"OCSPManagerHTTP-0","msg":"OCSP response","attr":{"status":{"code":0,"codeName":"OK"}}} {"t":{"$date":"2021-10-22T02:47:10.845+00:00"},"s":"I", "c":"NETWORK", "id":577164, "ctx":"OCSP Fetch and Staple","msg":"OCSP fetch/staple started"} {"t":{"$date":"2021-10-22T02:47:10.891+00:00"},"s":"I", "c":"NETWORK", "id":577165, "ctx":"OCSPManagerHTTP-0","msg":"OCSP fetch/staple completion"} {"t":{"$date":"2021-10-22T02:47:10.891+00:00"},"s":"I", "c":"NETWORK", "id":... and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) (warning: log line attempted (10.3k) over max size (10.0k), printing beginning and end) [2021-10-22T02:47:12.560+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:12.560] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:13.121+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:47:13.121] clusterConfig unchanged [2021-10-22T02:47:13.561+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:13.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:14.559+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:14.559] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:15.560+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:15.560] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:16.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:16.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:17.567+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:17.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:18.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:18.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:19.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:19.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:20.564+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:20.564] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:21.416+0000] [.info] [src/util/io.go:PreWarmDisk:56] [02:47:21.416] Pre-warming completed for disk /dev/nvme1n1 containing path /srv/mongodb/atlas-4wnh1p-config-0-node-1 after 12.978775184s [2021-10-22T02:47:21.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:21.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:22.561+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:22.561] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:23.212+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:47:23.212] clusterConfig unchanged [2021-10-22T02:47:23.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:23.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:24.562+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:24.562] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:25.563+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:25.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:26.563+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:26.563] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:27.564+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:27.564] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:27.574+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:47:27.574] SlowQueries Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:47:27.574+0000] [.info] [filebeat/mspooler/mspooler.go:waitForEvents:105] [02:47:27.574] LogIngestion Mspooler loop timeout after: 5m0s. Flushing to mms [2021-10-22T02:47:28.565+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:28.565] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:29.566+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:29.566] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:30.567+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:30.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:31.567+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:31.567] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:32.568+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:32.568] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:33.315+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:47:33.315] clusterConfig unchanged [2021-10-22T02:47:33.569+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:33.569] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:34.570+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:34.570] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:35.571+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:35.571] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:36.571+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:36.571] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:37.580+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:37.580] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:38.573+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:38.573] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:39.575+0000] [.info] [metrics/disk/disk_unix.go:runLsblkCommand:379] [02:47:39.575] lsblk cache invalidated, running lsblk command [2021-10-22T02:47:39.577+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:39.577] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:40.574+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:40.574] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:41.424+0000] [.info] [src/runtimestats/printer.go:mainLoop:58] [02:47:41.424] memory:rss=57323520 alloc=10925768 totalalloc=12756296736 mallocs=163038826 frees=162952803 heapinuse=17915904 heapobjects=86023 gcpausetotalns=1690993775 gcpauselastns=82534 gcnum=1830 utimens=289229847000 stimens=118360738000 minflt=698337 majflt=405 [2021-10-22T02:47:41.575+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:41.575] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:42.576+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:42.576] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:43.447+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:47:43.447] clusterConfig unchanged [2021-10-22T02:47:43.577+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:43.577] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:44.521+0000] [.info] [src/logrotator/mongologrotator.go:rotateLog:112] [02:47:44.521] Not rotating mongo logs for process = atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:42155 (local=false) because it is not up [2021-10-22T02:47:44.577+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:44.577] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:45.586+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:45.586] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:46.579+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:46.579] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:47.579+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:47.579] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:48.580+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:48.580] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:49.580+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:49.580] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:50.581+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:50.581] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:51.582+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:51.582] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:52.582+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:52.582] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:53.558+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:47:53.558] clusterConfig unchanged [2021-10-22T02:47:53.582+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:53.582] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:54.583+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:54.583] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:55.583+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:55.583] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:56.583+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:56.583] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:57.589+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:57.589] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:58.585+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:58.585] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:47:59.585+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:47:59.585] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:48:00.586+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:48:00.586] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:48:01.587+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:48:01.587] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:48:02.588+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:48:02.588] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:48:03.588+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:48:03.588] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:48:03.690+0000] [.info] [main/components/agent.go:LoadClusterConfig:265] [02:48:03.690] clusterConfig unchanged [2021-10-22T02:48:04.589+0000] [.error] [src/mongoctl/processctl.go:RunCommand:1083] [02:48:04.589] Server at atlas-4wnh1p-config-00-01.qzt8l.mmscloudteam.com:27017 (local=false) is down [2021-10-22T02:48:05.049+0000] [.info] [src/mmsjob/jobmanager.go:checkForNewJobs:316] [02:48:05.049] Received MMS Job : GET_MONGO_LOG_V2 (6172265ef350717484a08ca1) [2021-10-22T02:48:05.049+0000] [.info] [src/mmsjob/jobmanager.go:checkForNewJobs:316] [02:48:05.049] Received MMS Job : GET_MONGO_LOG_V2 (6172265ef350717484a08cb7) [2021-10-22T02:48:05.049+0000] [.info] [src/mmsjob/jobmanager.go:checkForNewJobs:316] [02:48:05.049] Received MMS Job : GET_MONGO_LOG_V2 (6172265ef350717484a08cc9) [2021-10-22T02:48:05.049+0000] [.info] [src/mmsjob/jobmanager.go:runJob:387] [02:48:05.049] Starting MMS Job : GET_MONGO_LOG_V2 (6172265ef350717484a08cc9) [2021-10-22T02:48:05.052+0000] [.info] [src/mmsjob/jobmanager.go:runJob:387] [02:48:05.052] Starting MMS Job : GET_MONGO_LOG_V2 (6172265ef350717484a08ca1) [2021-10-22T02:48:05.054+0000] [.info] [src/mmsjob/jobmanager.go:runJob:387] [02:48:05.054] Starting MMS Job : GET_MONGO_LOG_V2 (6172265ef350717484a08cb7) [2021-10-22T02:48:05.058+0000] [.info] [src/mmsjob/getmongohelpers.go:streamLogs:199] [02:48:05.058] Copied 1 log files to multipart file stream